diff options
Diffstat (limited to 'src/core/hle/kernel')
22 files changed, 874 insertions, 465 deletions
diff --git a/src/core/hle/kernel/k_auto_object.h b/src/core/hle/kernel/k_auto_object.h index 2827763d5..e8118c2b8 100644 --- a/src/core/hle/kernel/k_auto_object.h +++ b/src/core/hle/kernel/k_auto_object.h @@ -24,9 +24,7 @@ private: friend class ::Kernel::KClassTokenGenerator; \ static constexpr inline auto ObjectType = ::Kernel::KClassTokenGenerator::ObjectType::CLASS; \ static constexpr inline const char* const TypeName = #CLASS; \ - static constexpr inline ClassTokenType ClassToken() { \ - return ::Kernel::ClassToken<CLASS>; \ - } \ + static constexpr inline ClassTokenType ClassToken() { return ::Kernel::ClassToken<CLASS>; } \ \ public: \ YUZU_NON_COPYABLE(CLASS); \ @@ -37,15 +35,9 @@ public: constexpr ClassTokenType Token = ClassToken(); \ return TypeObj(TypeName, Token); \ } \ - static constexpr const char* GetStaticTypeName() { \ - return TypeName; \ - } \ - virtual TypeObj GetTypeObj() ATTRIBUTE { \ - return GetStaticTypeObj(); \ - } \ - virtual const char* GetTypeName() ATTRIBUTE { \ - return GetStaticTypeName(); \ - } \ + static constexpr const char* GetStaticTypeName() { return TypeName; } \ + virtual TypeObj GetTypeObj() ATTRIBUTE { return GetStaticTypeObj(); } \ + virtual const char* GetTypeName() ATTRIBUTE { return GetStaticTypeName(); } \ \ private: \ constexpr bool operator!=(const TypeObj& rhs) @@ -245,8 +237,8 @@ public: } template <typename U> - requires(std::derived_from<T, U> || - std::derived_from<U, T>) constexpr KScopedAutoObject(KScopedAutoObject<U>&& rhs) { + requires(std::derived_from<T, U> || std::derived_from<U, T>) + constexpr KScopedAutoObject(KScopedAutoObject<U>&& rhs) { if constexpr (std::derived_from<U, T>) { // Upcast. m_obj = rhs.m_obj; diff --git a/src/core/hle/kernel/k_code_memory.cpp b/src/core/hle/kernel/k_code_memory.cpp index 4b1c134d4..884eba001 100644 --- a/src/core/hle/kernel/k_code_memory.cpp +++ b/src/core/hle/kernel/k_code_memory.cpp @@ -27,13 +27,13 @@ Result KCodeMemory::Initialize(Core::DeviceMemory& device_memory, VAddr addr, si auto& page_table = m_owner->PageTable(); // Construct the page group. - m_page_group = {}; + m_page_group.emplace(kernel, page_table.GetBlockInfoManager()); // Lock the memory. - R_TRY(page_table.LockForCodeMemory(&m_page_group, addr, size)) + R_TRY(page_table.LockForCodeMemory(std::addressof(*m_page_group), addr, size)) // Clear the memory. - for (const auto& block : m_page_group.Nodes()) { + for (const auto& block : *m_page_group) { std::memset(device_memory.GetPointer<void>(block.GetAddress()), 0xFF, block.GetSize()); } @@ -51,12 +51,13 @@ Result KCodeMemory::Initialize(Core::DeviceMemory& device_memory, VAddr addr, si void KCodeMemory::Finalize() { // Unlock. if (!m_is_mapped && !m_is_owner_mapped) { - const size_t size = m_page_group.GetNumPages() * PageSize; - m_owner->PageTable().UnlockForCodeMemory(m_address, size, m_page_group); + const size_t size = m_page_group->GetNumPages() * PageSize; + m_owner->PageTable().UnlockForCodeMemory(m_address, size, *m_page_group); } // Close the page group. - m_page_group = {}; + m_page_group->Close(); + m_page_group->Finalize(); // Close our reference to our owner. m_owner->Close(); @@ -64,7 +65,7 @@ void KCodeMemory::Finalize() { Result KCodeMemory::Map(VAddr address, size_t size) { // Validate the size. - R_UNLESS(m_page_group.GetNumPages() == Common::DivideUp(size, PageSize), ResultInvalidSize); + R_UNLESS(m_page_group->GetNumPages() == Common::DivideUp(size, PageSize), ResultInvalidSize); // Lock ourselves. KScopedLightLock lk(m_lock); @@ -73,8 +74,8 @@ Result KCodeMemory::Map(VAddr address, size_t size) { R_UNLESS(!m_is_mapped, ResultInvalidState); // Map the memory. - R_TRY(kernel.CurrentProcess()->PageTable().MapPages( - address, m_page_group, KMemoryState::CodeOut, KMemoryPermission::UserReadWrite)); + R_TRY(kernel.CurrentProcess()->PageTable().MapPageGroup( + address, *m_page_group, KMemoryState::CodeOut, KMemoryPermission::UserReadWrite)); // Mark ourselves as mapped. m_is_mapped = true; @@ -84,14 +85,14 @@ Result KCodeMemory::Map(VAddr address, size_t size) { Result KCodeMemory::Unmap(VAddr address, size_t size) { // Validate the size. - R_UNLESS(m_page_group.GetNumPages() == Common::DivideUp(size, PageSize), ResultInvalidSize); + R_UNLESS(m_page_group->GetNumPages() == Common::DivideUp(size, PageSize), ResultInvalidSize); // Lock ourselves. KScopedLightLock lk(m_lock); // Unmap the memory. - R_TRY(kernel.CurrentProcess()->PageTable().UnmapPages(address, m_page_group, - KMemoryState::CodeOut)); + R_TRY(kernel.CurrentProcess()->PageTable().UnmapPageGroup(address, *m_page_group, + KMemoryState::CodeOut)); // Mark ourselves as unmapped. m_is_mapped = false; @@ -101,7 +102,7 @@ Result KCodeMemory::Unmap(VAddr address, size_t size) { Result KCodeMemory::MapToOwner(VAddr address, size_t size, Svc::MemoryPermission perm) { // Validate the size. - R_UNLESS(m_page_group.GetNumPages() == Common::DivideUp(size, PageSize), ResultInvalidSize); + R_UNLESS(m_page_group->GetNumPages() == Common::DivideUp(size, PageSize), ResultInvalidSize); // Lock ourselves. KScopedLightLock lk(m_lock); @@ -124,8 +125,8 @@ Result KCodeMemory::MapToOwner(VAddr address, size_t size, Svc::MemoryPermission } // Map the memory. - R_TRY( - m_owner->PageTable().MapPages(address, m_page_group, KMemoryState::GeneratedCode, k_perm)); + R_TRY(m_owner->PageTable().MapPageGroup(address, *m_page_group, KMemoryState::GeneratedCode, + k_perm)); // Mark ourselves as mapped. m_is_owner_mapped = true; @@ -135,13 +136,13 @@ Result KCodeMemory::MapToOwner(VAddr address, size_t size, Svc::MemoryPermission Result KCodeMemory::UnmapFromOwner(VAddr address, size_t size) { // Validate the size. - R_UNLESS(m_page_group.GetNumPages() == Common::DivideUp(size, PageSize), ResultInvalidSize); + R_UNLESS(m_page_group->GetNumPages() == Common::DivideUp(size, PageSize), ResultInvalidSize); // Lock ourselves. KScopedLightLock lk(m_lock); // Unmap the memory. - R_TRY(m_owner->PageTable().UnmapPages(address, m_page_group, KMemoryState::GeneratedCode)); + R_TRY(m_owner->PageTable().UnmapPageGroup(address, *m_page_group, KMemoryState::GeneratedCode)); // Mark ourselves as unmapped. m_is_owner_mapped = false; diff --git a/src/core/hle/kernel/k_code_memory.h b/src/core/hle/kernel/k_code_memory.h index 2e7e1436a..5b260b385 100644 --- a/src/core/hle/kernel/k_code_memory.h +++ b/src/core/hle/kernel/k_code_memory.h @@ -3,6 +3,8 @@ #pragma once +#include <optional> + #include "common/common_types.h" #include "core/device_memory.h" #include "core/hle/kernel/k_auto_object.h" @@ -49,11 +51,11 @@ public: return m_address; } size_t GetSize() const { - return m_is_initialized ? m_page_group.GetNumPages() * PageSize : 0; + return m_is_initialized ? m_page_group->GetNumPages() * PageSize : 0; } private: - KPageGroup m_page_group{}; + std::optional<KPageGroup> m_page_group{}; KProcess* m_owner{}; VAddr m_address{}; KLightLock m_lock; diff --git a/src/core/hle/kernel/k_condition_variable.cpp b/src/core/hle/kernel/k_condition_variable.cpp index 124149697..0c6b20db3 100644 --- a/src/core/hle/kernel/k_condition_variable.cpp +++ b/src/core/hle/kernel/k_condition_variable.cpp @@ -171,7 +171,7 @@ Result KConditionVariable::WaitForAddress(Handle handle, VAddr addr, u32 value) R_UNLESS(owner_thread != nullptr, ResultInvalidHandle); // Update the lock. - cur_thread->SetAddressKey(addr, value); + cur_thread->SetUserAddressKey(addr, value); owner_thread->AddWaiter(cur_thread); // Begin waiting. diff --git a/src/core/hle/kernel/k_hardware_timer.cpp b/src/core/hle/kernel/k_hardware_timer.cpp index 6bba79ea0..4dcd53821 100644 --- a/src/core/hle/kernel/k_hardware_timer.cpp +++ b/src/core/hle/kernel/k_hardware_timer.cpp @@ -18,7 +18,8 @@ void KHardwareTimer::Initialize() { } void KHardwareTimer::Finalize() { - this->DisableInterrupt(); + m_kernel.System().CoreTiming().UnscheduleEvent(m_event_type, reinterpret_cast<uintptr_t>(this)); + m_wakeup_time = std::numeric_limits<s64>::max(); m_event_type.reset(); } @@ -59,7 +60,8 @@ void KHardwareTimer::EnableInterrupt(s64 wakeup_time) { } void KHardwareTimer::DisableInterrupt() { - m_kernel.System().CoreTiming().UnscheduleEvent(m_event_type, reinterpret_cast<uintptr_t>(this)); + m_kernel.System().CoreTiming().UnscheduleEventWithoutWait(m_event_type, + reinterpret_cast<uintptr_t>(this)); m_wakeup_time = std::numeric_limits<s64>::max(); } diff --git a/src/core/hle/kernel/k_light_lock.cpp b/src/core/hle/kernel/k_light_lock.cpp index 43185320d..d791acbe3 100644 --- a/src/core/hle/kernel/k_light_lock.cpp +++ b/src/core/hle/kernel/k_light_lock.cpp @@ -68,7 +68,7 @@ bool KLightLock::LockSlowPath(uintptr_t _owner, uintptr_t _cur_thread) { // Add the current thread as a waiter on the owner. KThread* owner_thread = reinterpret_cast<KThread*>(_owner & ~1ULL); - cur_thread->SetAddressKey(reinterpret_cast<uintptr_t>(std::addressof(tag))); + cur_thread->SetKernelAddressKey(reinterpret_cast<uintptr_t>(std::addressof(tag))); owner_thread->AddWaiter(cur_thread); // Begin waiting to hold the lock. diff --git a/src/core/hle/kernel/k_memory_layout.h b/src/core/hle/kernel/k_memory_layout.h index fd6e1d3e6..17fa1a6ed 100644 --- a/src/core/hle/kernel/k_memory_layout.h +++ b/src/core/hle/kernel/k_memory_layout.h @@ -67,9 +67,9 @@ constexpr size_t KernelPageBufferAdditionalSize = 0x33C000; constexpr std::size_t KernelResourceSize = KernelPageTableHeapSize + KernelInitialPageHeapSize + KernelSlabHeapSize + KernelPageBufferHeapSize; -constexpr bool IsKernelAddressKey(VAddr key) { - return KernelVirtualAddressSpaceBase <= key && key <= KernelVirtualAddressSpaceLast; -} +//! NB: Use KThread::GetAddressKeyIsKernel(). +//! See explanation for deviation of GetAddressKey. +bool IsKernelAddressKey(VAddr key) = delete; constexpr bool IsKernelAddress(VAddr address) { return KernelVirtualAddressSpaceBase <= address && address < KernelVirtualAddressSpaceEnd; diff --git a/src/core/hle/kernel/k_memory_manager.cpp b/src/core/hle/kernel/k_memory_manager.cpp index bd33571da..cd6ea388e 100644 --- a/src/core/hle/kernel/k_memory_manager.cpp +++ b/src/core/hle/kernel/k_memory_manager.cpp @@ -223,7 +223,7 @@ Result KMemoryManager::AllocatePageGroupImpl(KPageGroup* out, size_t num_pages, // Ensure that we don't leave anything un-freed. ON_RESULT_FAILURE { - for (const auto& it : out->Nodes()) { + for (const auto& it : *out) { auto& manager = this->GetManager(it.GetAddress()); const size_t node_num_pages = std::min<u64>( it.GetNumPages(), (manager.GetEndAddress() - it.GetAddress()) / PageSize); @@ -285,7 +285,7 @@ Result KMemoryManager::AllocateAndOpen(KPageGroup* out, size_t num_pages, u32 op m_has_optimized_process[static_cast<size_t>(pool)], true)); // Open the first reference to the pages. - for (const auto& block : out->Nodes()) { + for (const auto& block : *out) { PAddr cur_address = block.GetAddress(); size_t remaining_pages = block.GetNumPages(); while (remaining_pages > 0) { @@ -335,7 +335,7 @@ Result KMemoryManager::AllocateForProcess(KPageGroup* out, size_t num_pages, u32 // Perform optimized memory tracking, if we should. if (optimized) { // Iterate over the allocated blocks. - for (const auto& block : out->Nodes()) { + for (const auto& block : *out) { // Get the block extents. const PAddr block_address = block.GetAddress(); const size_t block_pages = block.GetNumPages(); @@ -391,7 +391,7 @@ Result KMemoryManager::AllocateForProcess(KPageGroup* out, size_t num_pages, u32 } } else { // Set all the allocated memory. - for (const auto& block : out->Nodes()) { + for (const auto& block : *out) { std::memset(m_system.DeviceMemory().GetPointer<void>(block.GetAddress()), fill_pattern, block.GetSize()); } diff --git a/src/core/hle/kernel/k_page_group.cpp b/src/core/hle/kernel/k_page_group.cpp new file mode 100644 index 000000000..d8c644a33 --- /dev/null +++ b/src/core/hle/kernel/k_page_group.cpp @@ -0,0 +1,121 @@ +// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#include "core/hle/kernel/k_dynamic_resource_manager.h" +#include "core/hle/kernel/k_memory_manager.h" +#include "core/hle/kernel/k_page_group.h" +#include "core/hle/kernel/kernel.h" +#include "core/hle/kernel/svc_results.h" + +namespace Kernel { + +void KPageGroup::Finalize() { + KBlockInfo* cur = m_first_block; + while (cur != nullptr) { + KBlockInfo* next = cur->GetNext(); + m_manager->Free(cur); + cur = next; + } + + m_first_block = nullptr; + m_last_block = nullptr; +} + +void KPageGroup::CloseAndReset() { + auto& mm = m_kernel.MemoryManager(); + + KBlockInfo* cur = m_first_block; + while (cur != nullptr) { + KBlockInfo* next = cur->GetNext(); + mm.Close(cur->GetAddress(), cur->GetNumPages()); + m_manager->Free(cur); + cur = next; + } + + m_first_block = nullptr; + m_last_block = nullptr; +} + +size_t KPageGroup::GetNumPages() const { + size_t num_pages = 0; + + for (const auto& it : *this) { + num_pages += it.GetNumPages(); + } + + return num_pages; +} + +Result KPageGroup::AddBlock(KPhysicalAddress addr, size_t num_pages) { + // Succeed immediately if we're adding no pages. + R_SUCCEED_IF(num_pages == 0); + + // Check for overflow. + ASSERT(addr < addr + num_pages * PageSize); + + // Try to just append to the last block. + if (m_last_block != nullptr) { + R_SUCCEED_IF(m_last_block->TryConcatenate(addr, num_pages)); + } + + // Allocate a new block. + KBlockInfo* new_block = m_manager->Allocate(); + R_UNLESS(new_block != nullptr, ResultOutOfResource); + + // Initialize the block. + new_block->Initialize(addr, num_pages); + + // Add the block to our list. + if (m_last_block != nullptr) { + m_last_block->SetNext(new_block); + } else { + m_first_block = new_block; + } + m_last_block = new_block; + + R_SUCCEED(); +} + +void KPageGroup::Open() const { + auto& mm = m_kernel.MemoryManager(); + + for (const auto& it : *this) { + mm.Open(it.GetAddress(), it.GetNumPages()); + } +} + +void KPageGroup::OpenFirst() const { + auto& mm = m_kernel.MemoryManager(); + + for (const auto& it : *this) { + mm.OpenFirst(it.GetAddress(), it.GetNumPages()); + } +} + +void KPageGroup::Close() const { + auto& mm = m_kernel.MemoryManager(); + + for (const auto& it : *this) { + mm.Close(it.GetAddress(), it.GetNumPages()); + } +} + +bool KPageGroup::IsEquivalentTo(const KPageGroup& rhs) const { + auto lit = this->begin(); + auto rit = rhs.begin(); + auto lend = this->end(); + auto rend = rhs.end(); + + while (lit != lend && rit != rend) { + if (*lit != *rit) { + return false; + } + + ++lit; + ++rit; + } + + return lit == lend && rit == rend; +} + +} // namespace Kernel diff --git a/src/core/hle/kernel/k_page_group.h b/src/core/hle/kernel/k_page_group.h index 316f172f2..c07f17663 100644 --- a/src/core/hle/kernel/k_page_group.h +++ b/src/core/hle/kernel/k_page_group.h @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project +// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later #pragma once @@ -13,24 +13,23 @@ namespace Kernel { +class KBlockInfoManager; +class KernelCore; class KPageGroup; class KBlockInfo { -private: - friend class KPageGroup; - public: - constexpr KBlockInfo() = default; + constexpr explicit KBlockInfo() : m_next(nullptr) {} - constexpr void Initialize(PAddr addr, size_t np) { + constexpr void Initialize(KPhysicalAddress addr, size_t np) { ASSERT(Common::IsAligned(addr, PageSize)); ASSERT(static_cast<u32>(np) == np); - m_page_index = static_cast<u32>(addr) / PageSize; + m_page_index = static_cast<u32>(addr / PageSize); m_num_pages = static_cast<u32>(np); } - constexpr PAddr GetAddress() const { + constexpr KPhysicalAddress GetAddress() const { return m_page_index * PageSize; } constexpr size_t GetNumPages() const { @@ -39,10 +38,10 @@ public: constexpr size_t GetSize() const { return this->GetNumPages() * PageSize; } - constexpr PAddr GetEndAddress() const { + constexpr KPhysicalAddress GetEndAddress() const { return (m_page_index + m_num_pages) * PageSize; } - constexpr PAddr GetLastAddress() const { + constexpr KPhysicalAddress GetLastAddress() const { return this->GetEndAddress() - 1; } @@ -62,8 +61,8 @@ public: return !(*this == rhs); } - constexpr bool IsStrictlyBefore(PAddr addr) const { - const PAddr end = this->GetEndAddress(); + constexpr bool IsStrictlyBefore(KPhysicalAddress addr) const { + const KPhysicalAddress end = this->GetEndAddress(); if (m_page_index != 0 && end == 0) { return false; @@ -72,11 +71,11 @@ public: return end < addr; } - constexpr bool operator<(PAddr addr) const { + constexpr bool operator<(KPhysicalAddress addr) const { return this->IsStrictlyBefore(addr); } - constexpr bool TryConcatenate(PAddr addr, size_t np) { + constexpr bool TryConcatenate(KPhysicalAddress addr, size_t np) { if (addr != 0 && addr == this->GetEndAddress()) { m_num_pages += static_cast<u32>(np); return true; @@ -90,96 +89,118 @@ private: } private: + friend class KPageGroup; + KBlockInfo* m_next{}; u32 m_page_index{}; u32 m_num_pages{}; }; static_assert(sizeof(KBlockInfo) <= 0x10); -class KPageGroup final { +class KPageGroup { public: - class Node final { + class Iterator { public: - constexpr Node(u64 addr_, std::size_t num_pages_) : addr{addr_}, num_pages{num_pages_} {} + using iterator_category = std::forward_iterator_tag; + using value_type = const KBlockInfo; + using difference_type = std::ptrdiff_t; + using pointer = value_type*; + using reference = value_type&; + + constexpr explicit Iterator(pointer n) : m_node(n) {} + + constexpr bool operator==(const Iterator& rhs) const { + return m_node == rhs.m_node; + } + constexpr bool operator!=(const Iterator& rhs) const { + return !(*this == rhs); + } - constexpr u64 GetAddress() const { - return addr; + constexpr pointer operator->() const { + return m_node; + } + constexpr reference operator*() const { + return *m_node; } - constexpr std::size_t GetNumPages() const { - return num_pages; + constexpr Iterator& operator++() { + m_node = m_node->GetNext(); + return *this; } - constexpr std::size_t GetSize() const { - return GetNumPages() * PageSize; + constexpr Iterator operator++(int) { + const Iterator it{*this}; + ++(*this); + return it; } private: - u64 addr{}; - std::size_t num_pages{}; + pointer m_node{}; }; -public: - KPageGroup() = default; - KPageGroup(u64 address, u64 num_pages) { - ASSERT(AddBlock(address, num_pages).IsSuccess()); + explicit KPageGroup(KernelCore& kernel, KBlockInfoManager* m) + : m_kernel{kernel}, m_manager{m} {} + ~KPageGroup() { + this->Finalize(); } - constexpr std::list<Node>& Nodes() { - return nodes; - } + void CloseAndReset(); + void Finalize(); - constexpr const std::list<Node>& Nodes() const { - return nodes; + Iterator begin() const { + return Iterator{m_first_block}; + } + Iterator end() const { + return Iterator{nullptr}; + } + bool empty() const { + return m_first_block == nullptr; } - std::size_t GetNumPages() const { - std::size_t num_pages = 0; - for (const Node& node : nodes) { - num_pages += node.GetNumPages(); - } - return num_pages; - } - - bool IsEqual(KPageGroup& other) const { - auto this_node = nodes.begin(); - auto other_node = other.nodes.begin(); - while (this_node != nodes.end() && other_node != other.nodes.end()) { - if (this_node->GetAddress() != other_node->GetAddress() || - this_node->GetNumPages() != other_node->GetNumPages()) { - return false; - } - this_node = std::next(this_node); - other_node = std::next(other_node); - } + Result AddBlock(KPhysicalAddress addr, size_t num_pages); + void Open() const; + void OpenFirst() const; + void Close() const; + + size_t GetNumPages() const; + + bool IsEquivalentTo(const KPageGroup& rhs) const; + + bool operator==(const KPageGroup& rhs) const { + return this->IsEquivalentTo(rhs); + } - return this_node == nodes.end() && other_node == other.nodes.end(); + bool operator!=(const KPageGroup& rhs) const { + return !(*this == rhs); } - Result AddBlock(u64 address, u64 num_pages) { - if (!num_pages) { - return ResultSuccess; +private: + KernelCore& m_kernel; + KBlockInfo* m_first_block{}; + KBlockInfo* m_last_block{}; + KBlockInfoManager* m_manager{}; +}; + +class KScopedPageGroup { +public: + explicit KScopedPageGroup(const KPageGroup* gp) : m_pg(gp) { + if (m_pg) { + m_pg->Open(); } - if (!nodes.empty()) { - const auto node = nodes.back(); - if (node.GetAddress() + node.GetNumPages() * PageSize == address) { - address = node.GetAddress(); - num_pages += node.GetNumPages(); - nodes.pop_back(); - } + } + explicit KScopedPageGroup(const KPageGroup& gp) : KScopedPageGroup(std::addressof(gp)) {} + ~KScopedPageGroup() { + if (m_pg) { + m_pg->Close(); } - nodes.push_back({address, num_pages}); - return ResultSuccess; } - bool Empty() const { - return nodes.empty(); + void CancelClose() { + m_pg = nullptr; } - void Finalize() {} - private: - std::list<Node> nodes; + const KPageGroup* m_pg{}; }; } // namespace Kernel diff --git a/src/core/hle/kernel/k_page_table.cpp b/src/core/hle/kernel/k_page_table.cpp index 612fc76fa..2e13d5d0d 100644 --- a/src/core/hle/kernel/k_page_table.cpp +++ b/src/core/hle/kernel/k_page_table.cpp @@ -100,7 +100,7 @@ constexpr size_t GetAddressSpaceWidthFromType(FileSys::ProgramAddressSpaceType a KPageTable::KPageTable(Core::System& system_) : m_general_lock{system_.Kernel()}, - m_map_physical_memory_lock{system_.Kernel()}, m_system{system_} {} + m_map_physical_memory_lock{system_.Kernel()}, m_system{system_}, m_kernel{system_.Kernel()} {} KPageTable::~KPageTable() = default; @@ -373,7 +373,7 @@ Result KPageTable::MapProcessCode(VAddr addr, size_t num_pages, KMemoryState sta m_memory_block_slab_manager); // Allocate and open. - KPageGroup pg; + KPageGroup pg{m_kernel, m_block_info_manager}; R_TRY(m_system.Kernel().MemoryManager().AllocateAndOpen( &pg, num_pages, KMemoryManager::EncodeOption(KMemoryManager::Pool::Application, m_allocation_option))); @@ -432,9 +432,12 @@ Result KPageTable::MapCodeMemory(VAddr dst_address, VAddr src_address, size_t si const size_t num_pages = size / PageSize; // Create page groups for the memory being mapped. - KPageGroup pg; + KPageGroup pg{m_kernel, m_block_info_manager}; AddRegionToPages(src_address, num_pages, pg); + // We're going to perform an update, so create a helper. + KScopedPageTableUpdater updater(this); + // Reprotect the source as kernel-read/not mapped. const auto new_perm = static_cast<KMemoryPermission>(KMemoryPermission::KernelRead | KMemoryPermission::NotMapped); @@ -447,7 +450,10 @@ Result KPageTable::MapCodeMemory(VAddr dst_address, VAddr src_address, size_t si }); // Map the alias pages. - R_TRY(MapPages(dst_address, pg, new_perm)); + const KPageProperties dst_properties = {new_perm, false, false, + DisableMergeAttribute::DisableHead}; + R_TRY( + this->MapPageGroupImpl(updater.GetPageList(), dst_address, pg, dst_properties, false)); // We successfully mapped the alias pages, so we don't need to unprotect the src pages on // failure. @@ -593,7 +599,7 @@ Result KPageTable::MakePageGroup(KPageGroup& pg, VAddr addr, size_t num_pages) { const size_t size = num_pages * PageSize; // We're making a new group, not adding to an existing one. - R_UNLESS(pg.Empty(), ResultInvalidCurrentMemory); + R_UNLESS(pg.empty(), ResultInvalidCurrentMemory); // Begin traversal. Common::PageTable::TraversalContext context; @@ -640,11 +646,10 @@ Result KPageTable::MakePageGroup(KPageGroup& pg, VAddr addr, size_t num_pages) { R_SUCCEED(); } -bool KPageTable::IsValidPageGroup(const KPageGroup& pg_ll, VAddr addr, size_t num_pages) { +bool KPageTable::IsValidPageGroup(const KPageGroup& pg, VAddr addr, size_t num_pages) { ASSERT(this->IsLockedByCurrentThread()); const size_t size = num_pages * PageSize; - const auto& pg = pg_ll.Nodes(); const auto& memory_layout = m_system.Kernel().MemoryLayout(); // Empty groups are necessarily invalid. @@ -942,9 +947,6 @@ Result KPageTable::SetupForIpcServer(VAddr* out_addr, size_t size, VAddr src_add ON_RESULT_FAILURE { if (cur_mapped_addr != dst_addr) { - // HACK: Manually close the pages. - HACK_ClosePages(dst_addr, (cur_mapped_addr - dst_addr) / PageSize); - ASSERT(Operate(dst_addr, (cur_mapped_addr - dst_addr) / PageSize, KMemoryPermission::None, OperationType::Unmap) .IsSuccess()); @@ -1020,9 +1022,6 @@ Result KPageTable::SetupForIpcServer(VAddr* out_addr, size_t size, VAddr src_add // Map the page. R_TRY(Operate(cur_mapped_addr, 1, test_perm, OperationType::Map, start_partial_page)); - // HACK: Manually open the pages. - HACK_OpenPages(start_partial_page, 1); - // Update tracking extents. cur_mapped_addr += PageSize; cur_block_addr += PageSize; @@ -1051,9 +1050,6 @@ Result KPageTable::SetupForIpcServer(VAddr* out_addr, size_t size, VAddr src_add R_TRY(Operate(cur_mapped_addr, cur_block_size / PageSize, test_perm, OperationType::Map, cur_block_addr)); - // HACK: Manually open the pages. - HACK_OpenPages(cur_block_addr, cur_block_size / PageSize); - // Update tracking extents. cur_mapped_addr += cur_block_size; cur_block_addr = next_entry.phys_addr; @@ -1073,9 +1069,6 @@ Result KPageTable::SetupForIpcServer(VAddr* out_addr, size_t size, VAddr src_add R_TRY(Operate(cur_mapped_addr, last_block_size / PageSize, test_perm, OperationType::Map, cur_block_addr)); - // HACK: Manually open the pages. - HACK_OpenPages(cur_block_addr, last_block_size / PageSize); - // Update tracking extents. cur_mapped_addr += last_block_size; cur_block_addr += last_block_size; @@ -1107,9 +1100,6 @@ Result KPageTable::SetupForIpcServer(VAddr* out_addr, size_t size, VAddr src_add // Map the page. R_TRY(Operate(cur_mapped_addr, 1, test_perm, OperationType::Map, end_partial_page)); - - // HACK: Manually open the pages. - HACK_OpenPages(end_partial_page, 1); } // Update memory blocks to reflect our changes @@ -1211,9 +1201,6 @@ Result KPageTable::CleanupForIpcServer(VAddr address, size_t size, KMemoryState const size_t aligned_size = aligned_end - aligned_start; const size_t aligned_num_pages = aligned_size / PageSize; - // HACK: Manually close the pages. - HACK_ClosePages(aligned_start, aligned_num_pages); - // Unmap the pages. R_TRY(Operate(aligned_start, aligned_num_pages, KMemoryPermission::None, OperationType::Unmap)); @@ -1501,17 +1488,6 @@ void KPageTable::CleanupForIpcClientOnServerSetupFailure([[maybe_unused]] PageLi } } -void KPageTable::HACK_OpenPages(PAddr phys_addr, size_t num_pages) { - m_system.Kernel().MemoryManager().OpenFirst(phys_addr, num_pages); -} - -void KPageTable::HACK_ClosePages(VAddr virt_addr, size_t num_pages) { - for (size_t index = 0; index < num_pages; ++index) { - const auto paddr = GetPhysicalAddr(virt_addr + (index * PageSize)); - m_system.Kernel().MemoryManager().Close(paddr, 1); - } -} - Result KPageTable::MapPhysicalMemory(VAddr address, size_t size) { // Lock the physical memory lock. KScopedLightLock phys_lk(m_map_physical_memory_lock); @@ -1572,7 +1548,7 @@ Result KPageTable::MapPhysicalMemory(VAddr address, size_t size) { R_UNLESS(memory_reservation.Succeeded(), ResultLimitReached); // Allocate pages for the new memory. - KPageGroup pg; + KPageGroup pg{m_kernel, m_block_info_manager}; R_TRY(m_system.Kernel().MemoryManager().AllocateForProcess( &pg, (size - mapped_size) / PageSize, m_allocate_option, 0, 0)); @@ -1650,7 +1626,7 @@ Result KPageTable::MapPhysicalMemory(VAddr address, size_t size) { KScopedPageTableUpdater updater(this); // Prepare to iterate over the memory. - auto pg_it = pg.Nodes().begin(); + auto pg_it = pg.begin(); PAddr pg_phys_addr = pg_it->GetAddress(); size_t pg_pages = pg_it->GetNumPages(); @@ -1680,9 +1656,6 @@ Result KPageTable::MapPhysicalMemory(VAddr address, size_t size) { last_unmap_address + 1 - cur_address) / PageSize; - // HACK: Manually close the pages. - HACK_ClosePages(cur_address, cur_pages); - // Unmap. ASSERT(Operate(cur_address, cur_pages, KMemoryPermission::None, OperationType::Unmap) @@ -1703,7 +1676,7 @@ Result KPageTable::MapPhysicalMemory(VAddr address, size_t size) { // Release any remaining unmapped memory. m_system.Kernel().MemoryManager().OpenFirst(pg_phys_addr, pg_pages); m_system.Kernel().MemoryManager().Close(pg_phys_addr, pg_pages); - for (++pg_it; pg_it != pg.Nodes().end(); ++pg_it) { + for (++pg_it; pg_it != pg.end(); ++pg_it) { m_system.Kernel().MemoryManager().OpenFirst(pg_it->GetAddress(), pg_it->GetNumPages()); m_system.Kernel().MemoryManager().Close(pg_it->GetAddress(), @@ -1731,7 +1704,7 @@ Result KPageTable::MapPhysicalMemory(VAddr address, size_t size) { // Check if we're at the end of the physical block. if (pg_pages == 0) { // Ensure there are more pages to map. - ASSERT(pg_it != pg.Nodes().end()); + ASSERT(pg_it != pg.end()); // Advance our physical block. ++pg_it; @@ -1742,10 +1715,7 @@ Result KPageTable::MapPhysicalMemory(VAddr address, size_t size) { // Map whatever we can. const size_t cur_pages = std::min(pg_pages, map_pages); R_TRY(Operate(cur_address, cur_pages, KMemoryPermission::UserReadWrite, - OperationType::Map, pg_phys_addr)); - - // HACK: Manually open the pages. - HACK_OpenPages(pg_phys_addr, cur_pages); + OperationType::MapFirst, pg_phys_addr)); // Advance. cur_address += cur_pages * PageSize; @@ -1888,9 +1858,6 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, size_t size) { last_address + 1 - cur_address) / PageSize; - // HACK: Manually close the pages. - HACK_ClosePages(cur_address, cur_pages); - // Unmap. ASSERT(Operate(cur_address, cur_pages, KMemoryPermission::None, OperationType::Unmap) .IsSuccess()); @@ -1920,7 +1887,8 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, size_t size) { R_SUCCEED(); } -Result KPageTable::MapMemory(VAddr dst_address, VAddr src_address, size_t size) { +Result KPageTable::MapMemory(KProcessAddress dst_address, KProcessAddress src_address, + size_t size) { // Lock the table. KScopedLightLock lk(m_general_lock); @@ -1941,53 +1909,73 @@ Result KPageTable::MapMemory(VAddr dst_address, VAddr src_address, size_t size) KMemoryAttribute::None)); // Create an update allocator for the source. - Result src_allocator_result{ResultSuccess}; + Result src_allocator_result; KMemoryBlockManagerUpdateAllocator src_allocator(std::addressof(src_allocator_result), m_memory_block_slab_manager, num_src_allocator_blocks); R_TRY(src_allocator_result); // Create an update allocator for the destination. - Result dst_allocator_result{ResultSuccess}; + Result dst_allocator_result; KMemoryBlockManagerUpdateAllocator dst_allocator(std::addressof(dst_allocator_result), m_memory_block_slab_manager, num_dst_allocator_blocks); R_TRY(dst_allocator_result); // Map the memory. - KPageGroup page_linked_list; - const size_t num_pages{size / PageSize}; - const KMemoryPermission new_src_perm = static_cast<KMemoryPermission>( - KMemoryPermission::KernelRead | KMemoryPermission::NotMapped); - const KMemoryAttribute new_src_attr = KMemoryAttribute::Locked; - - AddRegionToPages(src_address, num_pages, page_linked_list); { + // Determine the number of pages being operated on. + const size_t num_pages = size / PageSize; + + // Create page groups for the memory being unmapped. + KPageGroup pg{m_kernel, m_block_info_manager}; + + // Create the page group representing the source. + R_TRY(this->MakePageGroup(pg, src_address, num_pages)); + + // We're going to perform an update, so create a helper. + KScopedPageTableUpdater updater(this); + // Reprotect the source as kernel-read/not mapped. - auto block_guard = detail::ScopeExit([&] { - Operate(src_address, num_pages, KMemoryPermission::UserReadWrite, - OperationType::ChangePermissions); - }); - R_TRY(Operate(src_address, num_pages, new_src_perm, OperationType::ChangePermissions)); - R_TRY(MapPages(dst_address, page_linked_list, KMemoryPermission::UserReadWrite)); + const KMemoryPermission new_src_perm = static_cast<KMemoryPermission>( + KMemoryPermission::KernelRead | KMemoryPermission::NotMapped); + const KMemoryAttribute new_src_attr = KMemoryAttribute::Locked; + const KPageProperties src_properties = {new_src_perm, false, false, + DisableMergeAttribute::DisableHeadBodyTail}; + R_TRY(this->Operate(src_address, num_pages, src_properties.perm, + OperationType::ChangePermissions)); - block_guard.Cancel(); - } + // Ensure that we unprotect the source pages on failure. + ON_RESULT_FAILURE { + const KPageProperties unprotect_properties = { + KMemoryPermission::UserReadWrite, false, false, + DisableMergeAttribute::EnableHeadBodyTail}; + ASSERT(this->Operate(src_address, num_pages, unprotect_properties.perm, + OperationType::ChangePermissions) == ResultSuccess); + }; - // Apply the memory block updates. - m_memory_block_manager.Update(std::addressof(src_allocator), src_address, num_pages, src_state, - new_src_perm, new_src_attr, - KMemoryBlockDisableMergeAttribute::Locked, - KMemoryBlockDisableMergeAttribute::None); - m_memory_block_manager.Update(std::addressof(dst_allocator), dst_address, num_pages, - KMemoryState::Stack, KMemoryPermission::UserReadWrite, - KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, - KMemoryBlockDisableMergeAttribute::None); + // Map the alias pages. + const KPageProperties dst_map_properties = {KMemoryPermission::UserReadWrite, false, false, + DisableMergeAttribute::DisableHead}; + R_TRY(this->MapPageGroupImpl(updater.GetPageList(), dst_address, pg, dst_map_properties, + false)); + + // Apply the memory block updates. + m_memory_block_manager.Update(std::addressof(src_allocator), src_address, num_pages, + src_state, new_src_perm, new_src_attr, + KMemoryBlockDisableMergeAttribute::Locked, + KMemoryBlockDisableMergeAttribute::None); + m_memory_block_manager.Update( + std::addressof(dst_allocator), dst_address, num_pages, KMemoryState::Stack, + KMemoryPermission::UserReadWrite, KMemoryAttribute::None, + KMemoryBlockDisableMergeAttribute::Normal, KMemoryBlockDisableMergeAttribute::None); + } R_SUCCEED(); } -Result KPageTable::UnmapMemory(VAddr dst_address, VAddr src_address, size_t size) { +Result KPageTable::UnmapMemory(KProcessAddress dst_address, KProcessAddress src_address, + size_t size) { // Lock the table. KScopedLightLock lk(m_general_lock); @@ -2009,108 +1997,208 @@ Result KPageTable::UnmapMemory(VAddr dst_address, VAddr src_address, size_t size KMemoryPermission::None, KMemoryAttribute::All, KMemoryAttribute::None)); // Create an update allocator for the source. - Result src_allocator_result{ResultSuccess}; + Result src_allocator_result; KMemoryBlockManagerUpdateAllocator src_allocator(std::addressof(src_allocator_result), m_memory_block_slab_manager, num_src_allocator_blocks); R_TRY(src_allocator_result); // Create an update allocator for the destination. - Result dst_allocator_result{ResultSuccess}; + Result dst_allocator_result; KMemoryBlockManagerUpdateAllocator dst_allocator(std::addressof(dst_allocator_result), m_memory_block_slab_manager, num_dst_allocator_blocks); R_TRY(dst_allocator_result); - KPageGroup src_pages; - KPageGroup dst_pages; - const size_t num_pages{size / PageSize}; + // Unmap the memory. + { + // Determine the number of pages being operated on. + const size_t num_pages = size / PageSize; - AddRegionToPages(src_address, num_pages, src_pages); - AddRegionToPages(dst_address, num_pages, dst_pages); + // Create page groups for the memory being unmapped. + KPageGroup pg{m_kernel, m_block_info_manager}; - R_UNLESS(dst_pages.IsEqual(src_pages), ResultInvalidMemoryRegion); + // Create the page group representing the destination. + R_TRY(this->MakePageGroup(pg, dst_address, num_pages)); - { - auto block_guard = detail::ScopeExit([&] { MapPages(dst_address, dst_pages, dst_perm); }); + // Ensure the page group is the valid for the source. + R_UNLESS(this->IsValidPageGroup(pg, src_address, num_pages), ResultInvalidMemoryRegion); - R_TRY(Operate(dst_address, num_pages, KMemoryPermission::None, OperationType::Unmap)); - R_TRY(Operate(src_address, num_pages, KMemoryPermission::UserReadWrite, - OperationType::ChangePermissions)); + // We're going to perform an update, so create a helper. + KScopedPageTableUpdater updater(this); - block_guard.Cancel(); - } + // Unmap the aliased copy of the pages. + const KPageProperties dst_unmap_properties = {KMemoryPermission::None, false, false, + DisableMergeAttribute::None}; + R_TRY( + this->Operate(dst_address, num_pages, dst_unmap_properties.perm, OperationType::Unmap)); + + // Ensure that we re-map the aliased pages on failure. + ON_RESULT_FAILURE { + this->RemapPageGroup(updater.GetPageList(), dst_address, size, pg); + }; - // Apply the memory block updates. - m_memory_block_manager.Update(std::addressof(src_allocator), src_address, num_pages, src_state, - KMemoryPermission::UserReadWrite, KMemoryAttribute::None, - KMemoryBlockDisableMergeAttribute::None, - KMemoryBlockDisableMergeAttribute::Locked); - m_memory_block_manager.Update(std::addressof(dst_allocator), dst_address, num_pages, - KMemoryState::None, KMemoryPermission::None, - KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::None, - KMemoryBlockDisableMergeAttribute::Normal); + // Try to set the permissions for the source pages back to what they should be. + const KPageProperties src_properties = {KMemoryPermission::UserReadWrite, false, false, + DisableMergeAttribute::EnableAndMergeHeadBodyTail}; + R_TRY(this->Operate(src_address, num_pages, src_properties.perm, + OperationType::ChangePermissions)); + + // Apply the memory block updates. + m_memory_block_manager.Update( + std::addressof(src_allocator), src_address, num_pages, src_state, + KMemoryPermission::UserReadWrite, KMemoryAttribute::None, + KMemoryBlockDisableMergeAttribute::None, KMemoryBlockDisableMergeAttribute::Locked); + m_memory_block_manager.Update( + std::addressof(dst_allocator), dst_address, num_pages, KMemoryState::None, + KMemoryPermission::None, KMemoryAttribute::None, + KMemoryBlockDisableMergeAttribute::None, KMemoryBlockDisableMergeAttribute::Normal); + } R_SUCCEED(); } -Result KPageTable::MapPages(VAddr addr, const KPageGroup& page_linked_list, - KMemoryPermission perm) { +Result KPageTable::AllocateAndMapPagesImpl(PageLinkedList* page_list, KProcessAddress address, + size_t num_pages, KMemoryPermission perm) { ASSERT(this->IsLockedByCurrentThread()); - VAddr cur_addr{addr}; + // Create a page group to hold the pages we allocate. + KPageGroup pg{m_kernel, m_block_info_manager}; - for (const auto& node : page_linked_list.Nodes()) { - if (const auto result{ - Operate(cur_addr, node.GetNumPages(), perm, OperationType::Map, node.GetAddress())}; - result.IsError()) { - const size_t num_pages{(addr - cur_addr) / PageSize}; + // Allocate the pages. + R_TRY( + m_kernel.MemoryManager().AllocateAndOpen(std::addressof(pg), num_pages, m_allocate_option)); - ASSERT(Operate(addr, num_pages, KMemoryPermission::None, OperationType::Unmap) - .IsSuccess()); + // Ensure that the page group is closed when we're done working with it. + SCOPE_EXIT({ pg.Close(); }); + + // Clear all pages. + for (const auto& it : pg) { + std::memset(m_system.DeviceMemory().GetPointer<void>(it.GetAddress()), m_heap_fill_value, + it.GetSize()); + } + + // Map the pages. + R_RETURN(this->Operate(address, num_pages, pg, OperationType::MapGroup)); +} + +Result KPageTable::MapPageGroupImpl(PageLinkedList* page_list, KProcessAddress address, + const KPageGroup& pg, const KPageProperties properties, + bool reuse_ll) { + ASSERT(this->IsLockedByCurrentThread()); + + // Note the current address, so that we can iterate. + const KProcessAddress start_address = address; + KProcessAddress cur_address = address; - R_RETURN(result); + // Ensure that we clean up on failure. + ON_RESULT_FAILURE { + ASSERT(!reuse_ll); + if (cur_address != start_address) { + const KPageProperties unmap_properties = {KMemoryPermission::None, false, false, + DisableMergeAttribute::None}; + ASSERT(this->Operate(start_address, (cur_address - start_address) / PageSize, + unmap_properties.perm, OperationType::Unmap) == ResultSuccess); } + }; - cur_addr += node.GetNumPages() * PageSize; + // Iterate, mapping all pages in the group. + for (const auto& block : pg) { + // Map and advance. + const KPageProperties cur_properties = + (cur_address == start_address) + ? properties + : KPageProperties{properties.perm, properties.io, properties.uncached, + DisableMergeAttribute::None}; + this->Operate(cur_address, block.GetNumPages(), cur_properties.perm, OperationType::Map, + block.GetAddress()); + cur_address += block.GetSize(); } + // We succeeded! R_SUCCEED(); } -Result KPageTable::MapPages(VAddr address, KPageGroup& page_linked_list, KMemoryState state, - KMemoryPermission perm) { - // Check that the map is in range. - const size_t num_pages{page_linked_list.GetNumPages()}; - const size_t size{num_pages * PageSize}; - R_UNLESS(this->CanContain(address, size, state), ResultInvalidCurrentMemory); +void KPageTable::RemapPageGroup(PageLinkedList* page_list, KProcessAddress address, size_t size, + const KPageGroup& pg) { + ASSERT(this->IsLockedByCurrentThread()); - // Lock the table. - KScopedLightLock lk(m_general_lock); + // Note the current address, so that we can iterate. + const KProcessAddress start_address = address; + const KProcessAddress last_address = start_address + size - 1; + const KProcessAddress end_address = last_address + 1; - // Check the memory state. - R_TRY(this->CheckMemoryState(address, size, KMemoryState::All, KMemoryState::Free, - KMemoryPermission::None, KMemoryPermission::None, - KMemoryAttribute::None, KMemoryAttribute::None)); + // Iterate over the memory. + auto pg_it = pg.begin(); + ASSERT(pg_it != pg.end()); - // Create an update allocator. - Result allocator_result{ResultSuccess}; - KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), - m_memory_block_slab_manager); + KPhysicalAddress pg_phys_addr = pg_it->GetAddress(); + size_t pg_pages = pg_it->GetNumPages(); - // Map the pages. - R_TRY(MapPages(address, page_linked_list, perm)); + auto it = m_memory_block_manager.FindIterator(start_address); + while (true) { + // Check that the iterator is valid. + ASSERT(it != m_memory_block_manager.end()); - // Update the blocks. - m_memory_block_manager.Update(std::addressof(allocator), address, num_pages, state, perm, - KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, - KMemoryBlockDisableMergeAttribute::None); + // Get the memory info. + const KMemoryInfo info = it->GetMemoryInfo(); - R_SUCCEED(); + // Determine the range to map. + KProcessAddress map_address = std::max<VAddr>(info.GetAddress(), start_address); + const KProcessAddress map_end_address = std::min<VAddr>(info.GetEndAddress(), end_address); + ASSERT(map_end_address != map_address); + + // Determine if we should disable head merge. + const bool disable_head_merge = + info.GetAddress() >= start_address && + True(info.GetDisableMergeAttribute() & KMemoryBlockDisableMergeAttribute::Normal); + const KPageProperties map_properties = { + info.GetPermission(), false, false, + disable_head_merge ? DisableMergeAttribute::DisableHead : DisableMergeAttribute::None}; + + // While we have pages to map, map them. + size_t map_pages = (map_end_address - map_address) / PageSize; + while (map_pages > 0) { + // Check if we're at the end of the physical block. + if (pg_pages == 0) { + // Ensure there are more pages to map. + ASSERT(pg_it != pg.end()); + + // Advance our physical block. + ++pg_it; + pg_phys_addr = pg_it->GetAddress(); + pg_pages = pg_it->GetNumPages(); + } + + // Map whatever we can. + const size_t cur_pages = std::min(pg_pages, map_pages); + ASSERT(this->Operate(map_address, map_pages, map_properties.perm, OperationType::Map, + pg_phys_addr) == ResultSuccess); + + // Advance. + map_address += cur_pages * PageSize; + map_pages -= cur_pages; + + pg_phys_addr += cur_pages * PageSize; + pg_pages -= cur_pages; + } + + // Check if we're done. + if (last_address <= info.GetLastAddress()) { + break; + } + + // Advance. + ++it; + } + + // Check that we re-mapped precisely the page group. + ASSERT((++pg_it) == pg.end()); } -Result KPageTable::MapPages(VAddr* out_addr, size_t num_pages, size_t alignment, PAddr phys_addr, - bool is_pa_valid, VAddr region_start, size_t region_num_pages, +Result KPageTable::MapPages(KProcessAddress* out_addr, size_t num_pages, size_t alignment, + KPhysicalAddress phys_addr, bool is_pa_valid, + KProcessAddress region_start, size_t region_num_pages, KMemoryState state, KMemoryPermission perm) { ASSERT(Common::IsAligned(alignment, PageSize) && alignment >= PageSize); @@ -2123,26 +2211,30 @@ Result KPageTable::MapPages(VAddr* out_addr, size_t num_pages, size_t alignment, KScopedLightLock lk(m_general_lock); // Find a random address to map at. - VAddr addr = this->FindFreeArea(region_start, region_num_pages, num_pages, alignment, 0, - this->GetNumGuardPages()); + KProcessAddress addr = this->FindFreeArea(region_start, region_num_pages, num_pages, alignment, + 0, this->GetNumGuardPages()); R_UNLESS(addr != 0, ResultOutOfMemory); ASSERT(Common::IsAligned(addr, alignment)); ASSERT(this->CanContain(addr, num_pages * PageSize, state)); ASSERT(this->CheckMemoryState(addr, num_pages * PageSize, KMemoryState::All, KMemoryState::Free, KMemoryPermission::None, KMemoryPermission::None, - KMemoryAttribute::None, KMemoryAttribute::None) - .IsSuccess()); + KMemoryAttribute::None, KMemoryAttribute::None) == ResultSuccess); // Create an update allocator. - Result allocator_result{ResultSuccess}; + Result allocator_result; KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), m_memory_block_slab_manager); + R_TRY(allocator_result); + + // We're going to perform an update, so create a helper. + KScopedPageTableUpdater updater(this); // Perform mapping operation. if (is_pa_valid) { - R_TRY(this->Operate(addr, num_pages, perm, OperationType::Map, phys_addr)); + const KPageProperties properties = {perm, false, false, DisableMergeAttribute::DisableHead}; + R_TRY(this->Operate(addr, num_pages, properties.perm, OperationType::Map, phys_addr)); } else { - UNIMPLEMENTED(); + R_TRY(this->AllocateAndMapPagesImpl(updater.GetPageList(), addr, num_pages, perm)); } // Update the blocks. @@ -2155,28 +2247,45 @@ Result KPageTable::MapPages(VAddr* out_addr, size_t num_pages, size_t alignment, R_SUCCEED(); } -Result KPageTable::UnmapPages(VAddr addr, const KPageGroup& page_linked_list) { - ASSERT(this->IsLockedByCurrentThread()); +Result KPageTable::MapPages(KProcessAddress address, size_t num_pages, KMemoryState state, + KMemoryPermission perm) { + // Check that the map is in range. + const size_t size = num_pages * PageSize; + R_UNLESS(this->CanContain(address, size, state), ResultInvalidCurrentMemory); - VAddr cur_addr{addr}; + // Lock the table. + KScopedLightLock lk(m_general_lock); - for (const auto& node : page_linked_list.Nodes()) { - if (const auto result{Operate(cur_addr, node.GetNumPages(), KMemoryPermission::None, - OperationType::Unmap)}; - result.IsError()) { - R_RETURN(result); - } + // Check the memory state. + size_t num_allocator_blocks; + R_TRY(this->CheckMemoryState(std::addressof(num_allocator_blocks), address, size, + KMemoryState::All, KMemoryState::Free, KMemoryPermission::None, + KMemoryPermission::None, KMemoryAttribute::None, + KMemoryAttribute::None)); - cur_addr += node.GetNumPages() * PageSize; - } + // Create an update allocator. + Result allocator_result; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + m_memory_block_slab_manager, num_allocator_blocks); + R_TRY(allocator_result); + + // We're going to perform an update, so create a helper. + KScopedPageTableUpdater updater(this); + + // Map the pages. + R_TRY(this->AllocateAndMapPagesImpl(updater.GetPageList(), address, num_pages, perm)); + + // Update the blocks. + m_memory_block_manager.Update(std::addressof(allocator), address, num_pages, state, perm, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, + KMemoryBlockDisableMergeAttribute::None); R_SUCCEED(); } -Result KPageTable::UnmapPages(VAddr address, KPageGroup& page_linked_list, KMemoryState state) { +Result KPageTable::UnmapPages(KProcessAddress address, size_t num_pages, KMemoryState state) { // Check that the unmap is in range. - const size_t num_pages{page_linked_list.GetNumPages()}; - const size_t size{num_pages * PageSize}; + const size_t size = num_pages * PageSize; R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory); // Lock the table. @@ -2190,13 +2299,18 @@ Result KPageTable::UnmapPages(VAddr address, KPageGroup& page_linked_list, KMemo KMemoryAttribute::None)); // Create an update allocator. - Result allocator_result{ResultSuccess}; + Result allocator_result; KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), m_memory_block_slab_manager, num_allocator_blocks); R_TRY(allocator_result); + // We're going to perform an update, so create a helper. + KScopedPageTableUpdater updater(this); + // Perform the unmap. - R_TRY(UnmapPages(address, page_linked_list)); + const KPageProperties unmap_properties = {KMemoryPermission::None, false, false, + DisableMergeAttribute::None}; + R_TRY(this->Operate(address, num_pages, unmap_properties.perm, OperationType::Unmap)); // Update the blocks. m_memory_block_manager.Update(std::addressof(allocator), address, num_pages, KMemoryState::Free, @@ -2207,29 +2321,130 @@ Result KPageTable::UnmapPages(VAddr address, KPageGroup& page_linked_list, KMemo R_SUCCEED(); } -Result KPageTable::UnmapPages(VAddr address, size_t num_pages, KMemoryState state) { - // Check that the unmap is in range. +Result KPageTable::MapPageGroup(KProcessAddress* out_addr, const KPageGroup& pg, + KProcessAddress region_start, size_t region_num_pages, + KMemoryState state, KMemoryPermission perm) { + ASSERT(!this->IsLockedByCurrentThread()); + + // Ensure this is a valid map request. + const size_t num_pages = pg.GetNumPages(); + R_UNLESS(this->CanContain(region_start, region_num_pages * PageSize, state), + ResultInvalidCurrentMemory); + R_UNLESS(num_pages < region_num_pages, ResultOutOfMemory); + + // Lock the table. + KScopedLightLock lk(m_general_lock); + + // Find a random address to map at. + KProcessAddress addr = this->FindFreeArea(region_start, region_num_pages, num_pages, PageSize, + 0, this->GetNumGuardPages()); + R_UNLESS(addr != 0, ResultOutOfMemory); + ASSERT(this->CanContain(addr, num_pages * PageSize, state)); + ASSERT(this->CheckMemoryState(addr, num_pages * PageSize, KMemoryState::All, KMemoryState::Free, + KMemoryPermission::None, KMemoryPermission::None, + KMemoryAttribute::None, KMemoryAttribute::None) == ResultSuccess); + + // Create an update allocator. + Result allocator_result; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + m_memory_block_slab_manager); + R_TRY(allocator_result); + + // We're going to perform an update, so create a helper. + KScopedPageTableUpdater updater(this); + + // Perform mapping operation. + const KPageProperties properties = {perm, state == KMemoryState::Io, false, + DisableMergeAttribute::DisableHead}; + R_TRY(this->MapPageGroupImpl(updater.GetPageList(), addr, pg, properties, false)); + + // Update the blocks. + m_memory_block_manager.Update(std::addressof(allocator), addr, num_pages, state, perm, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, + KMemoryBlockDisableMergeAttribute::None); + + // We successfully mapped the pages. + *out_addr = addr; + R_SUCCEED(); +} + +Result KPageTable::MapPageGroup(KProcessAddress addr, const KPageGroup& pg, KMemoryState state, + KMemoryPermission perm) { + ASSERT(!this->IsLockedByCurrentThread()); + + // Ensure this is a valid map request. + const size_t num_pages = pg.GetNumPages(); const size_t size = num_pages * PageSize; - R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory); + R_UNLESS(this->CanContain(addr, size, state), ResultInvalidCurrentMemory); // Lock the table. KScopedLightLock lk(m_general_lock); - // Check the memory state. - size_t num_allocator_blocks{}; + // Check if state allows us to map. + size_t num_allocator_blocks; + R_TRY(this->CheckMemoryState(std::addressof(num_allocator_blocks), addr, size, + KMemoryState::All, KMemoryState::Free, KMemoryPermission::None, + KMemoryPermission::None, KMemoryAttribute::None, + KMemoryAttribute::None)); + + // Create an update allocator. + Result allocator_result; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + m_memory_block_slab_manager, num_allocator_blocks); + R_TRY(allocator_result); + + // We're going to perform an update, so create a helper. + KScopedPageTableUpdater updater(this); + + // Perform mapping operation. + const KPageProperties properties = {perm, state == KMemoryState::Io, false, + DisableMergeAttribute::DisableHead}; + R_TRY(this->MapPageGroupImpl(updater.GetPageList(), addr, pg, properties, false)); + + // Update the blocks. + m_memory_block_manager.Update(std::addressof(allocator), addr, num_pages, state, perm, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, + KMemoryBlockDisableMergeAttribute::None); + + // We successfully mapped the pages. + R_SUCCEED(); +} + +Result KPageTable::UnmapPageGroup(KProcessAddress address, const KPageGroup& pg, + KMemoryState state) { + ASSERT(!this->IsLockedByCurrentThread()); + + // Ensure this is a valid unmap request. + const size_t num_pages = pg.GetNumPages(); + const size_t size = num_pages * PageSize; + R_UNLESS(this->CanContain(address, size, state), ResultInvalidCurrentMemory); + + // Lock the table. + KScopedLightLock lk(m_general_lock); + + // Check if state allows us to unmap. + size_t num_allocator_blocks; R_TRY(this->CheckMemoryState(std::addressof(num_allocator_blocks), address, size, KMemoryState::All, state, KMemoryPermission::None, KMemoryPermission::None, KMemoryAttribute::All, KMemoryAttribute::None)); + // Check that the page group is valid. + R_UNLESS(this->IsValidPageGroup(pg, address, num_pages), ResultInvalidCurrentMemory); + // Create an update allocator. - Result allocator_result{ResultSuccess}; + Result allocator_result; KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), m_memory_block_slab_manager, num_allocator_blocks); R_TRY(allocator_result); - // Perform the unmap. - R_TRY(Operate(address, num_pages, KMemoryPermission::None, OperationType::Unmap)); + // We're going to perform an update, so create a helper. + KScopedPageTableUpdater updater(this); + + // Perform unmapping operation. + const KPageProperties properties = {KMemoryPermission::None, false, false, + DisableMergeAttribute::None}; + R_TRY(this->Operate(address, num_pages, properties.perm, OperationType::Unmap)); // Update the blocks. m_memory_block_manager.Update(std::addressof(allocator), address, num_pages, KMemoryState::Free, @@ -2527,13 +2742,13 @@ Result KPageTable::SetHeapSize(VAddr* out, size_t size) { R_UNLESS(memory_reservation.Succeeded(), ResultLimitReached); // Allocate pages for the heap extension. - KPageGroup pg; + KPageGroup pg{m_kernel, m_block_info_manager}; R_TRY(m_system.Kernel().MemoryManager().AllocateAndOpen( &pg, allocation_size / PageSize, KMemoryManager::EncodeOption(m_memory_pool, m_allocation_option))); // Clear all the newly allocated pages. - for (const auto& it : pg.Nodes()) { + for (const auto& it : pg) { std::memset(m_system.DeviceMemory().GetPointer<void>(it.GetAddress()), m_heap_fill_value, it.GetSize()); } @@ -2589,42 +2804,6 @@ Result KPageTable::SetHeapSize(VAddr* out, size_t size) { } } -ResultVal<VAddr> KPageTable::AllocateAndMapMemory(size_t needed_num_pages, size_t align, - bool is_map_only, VAddr region_start, - size_t region_num_pages, KMemoryState state, - KMemoryPermission perm, PAddr map_addr) { - KScopedLightLock lk(m_general_lock); - - R_UNLESS(CanContain(region_start, region_num_pages * PageSize, state), - ResultInvalidCurrentMemory); - R_UNLESS(region_num_pages > needed_num_pages, ResultOutOfMemory); - const VAddr addr{ - AllocateVirtualMemory(region_start, region_num_pages, needed_num_pages, align)}; - R_UNLESS(addr, ResultOutOfMemory); - - // Create an update allocator. - Result allocator_result{ResultSuccess}; - KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), - m_memory_block_slab_manager); - - if (is_map_only) { - R_TRY(Operate(addr, needed_num_pages, perm, OperationType::Map, map_addr)); - } else { - KPageGroup page_group; - R_TRY(m_system.Kernel().MemoryManager().AllocateForProcess( - &page_group, needed_num_pages, - KMemoryManager::EncodeOption(m_memory_pool, m_allocation_option), 0, 0)); - R_TRY(Operate(addr, needed_num_pages, page_group, OperationType::MapGroup)); - } - - // Update the blocks. - m_memory_block_manager.Update(std::addressof(allocator), addr, needed_num_pages, state, perm, - KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, - KMemoryBlockDisableMergeAttribute::None); - - return addr; -} - Result KPageTable::LockForMapDeviceAddressSpace(bool* out_is_io, VAddr address, size_t size, KMemoryPermission perm, bool is_aligned, bool check_heap) { @@ -2795,19 +2974,28 @@ Result KPageTable::Operate(VAddr addr, size_t num_pages, const KPageGroup& page_ ASSERT(num_pages > 0); ASSERT(num_pages == page_group.GetNumPages()); - for (const auto& node : page_group.Nodes()) { - const size_t size{node.GetNumPages() * PageSize}; + switch (operation) { + case OperationType::MapGroup: { + // We want to maintain a new reference to every page in the group. + KScopedPageGroup spg(page_group); + + for (const auto& node : page_group) { + const size_t size{node.GetNumPages() * PageSize}; - switch (operation) { - case OperationType::MapGroup: + // Map the pages. m_system.Memory().MapMemoryRegion(*m_page_table_impl, addr, size, node.GetAddress()); - break; - default: - ASSERT(false); - break; + + addr += size; } - addr += size; + // We succeeded! We want to persist the reference to the pages. + spg.CancelClose(); + + break; + } + default: + ASSERT(false); + break; } R_SUCCEED(); @@ -2822,13 +3010,29 @@ Result KPageTable::Operate(VAddr addr, size_t num_pages, KMemoryPermission perm, ASSERT(ContainsPages(addr, num_pages)); switch (operation) { - case OperationType::Unmap: + case OperationType::Unmap: { + // Ensure that any pages we track close on exit. + KPageGroup pages_to_close{m_kernel, this->GetBlockInfoManager()}; + SCOPE_EXIT({ pages_to_close.CloseAndReset(); }); + + this->AddRegionToPages(addr, num_pages, pages_to_close); m_system.Memory().UnmapRegion(*m_page_table_impl, addr, num_pages * PageSize); break; + } + case OperationType::MapFirst: case OperationType::Map: { ASSERT(map_addr); ASSERT(Common::IsAligned(map_addr, PageSize)); m_system.Memory().MapMemoryRegion(*m_page_table_impl, addr, num_pages * PageSize, map_addr); + + // Open references to pages, if we should. + if (IsHeapPhysicalAddress(m_kernel.MemoryLayout(), map_addr)) { + if (operation == OperationType::MapFirst) { + m_kernel.MemoryManager().OpenFirst(map_addr, num_pages); + } else { + m_kernel.MemoryManager().Open(map_addr, num_pages); + } + } break; } case OperationType::Separate: { diff --git a/src/core/hle/kernel/k_page_table.h b/src/core/hle/kernel/k_page_table.h index f1ca785d7..367dab613 100644 --- a/src/core/hle/kernel/k_page_table.h +++ b/src/core/hle/kernel/k_page_table.h @@ -24,12 +24,36 @@ class System; namespace Kernel { +enum class DisableMergeAttribute : u8 { + None = (0U << 0), + DisableHead = (1U << 0), + DisableHeadAndBody = (1U << 1), + EnableHeadAndBody = (1U << 2), + DisableTail = (1U << 3), + EnableTail = (1U << 4), + EnableAndMergeHeadBodyTail = (1U << 5), + EnableHeadBodyTail = EnableHeadAndBody | EnableTail, + DisableHeadBodyTail = DisableHeadAndBody | DisableTail, +}; + +struct KPageProperties { + KMemoryPermission perm; + bool io; + bool uncached; + DisableMergeAttribute disable_merge_attributes; +}; +static_assert(std::is_trivial_v<KPageProperties>); +static_assert(sizeof(KPageProperties) == sizeof(u32)); + class KBlockInfoManager; class KMemoryBlockManager; class KResourceLimit; class KSystemResource; class KPageTable final { +protected: + struct PageLinkedList; + public: enum class ICacheInvalidationStrategy : u32 { InvalidateRange, InvalidateAll }; @@ -57,27 +81,12 @@ public: Result UnmapPhysicalMemory(VAddr addr, size_t size); Result MapMemory(VAddr dst_addr, VAddr src_addr, size_t size); Result UnmapMemory(VAddr dst_addr, VAddr src_addr, size_t size); - Result MapPages(VAddr addr, KPageGroup& page_linked_list, KMemoryState state, - KMemoryPermission perm); - Result MapPages(VAddr* out_addr, size_t num_pages, size_t alignment, PAddr phys_addr, - KMemoryState state, KMemoryPermission perm) { - R_RETURN(this->MapPages(out_addr, num_pages, alignment, phys_addr, true, - this->GetRegionAddress(state), - this->GetRegionSize(state) / PageSize, state, perm)); - } - Result UnmapPages(VAddr addr, KPageGroup& page_linked_list, KMemoryState state); - Result UnmapPages(VAddr address, size_t num_pages, KMemoryState state); Result SetProcessMemoryPermission(VAddr addr, size_t size, Svc::MemoryPermission svc_perm); KMemoryInfo QueryInfo(VAddr addr); Result SetMemoryPermission(VAddr addr, size_t size, Svc::MemoryPermission perm); Result SetMemoryAttribute(VAddr addr, size_t size, u32 mask, u32 attr); Result SetMaxHeapSize(size_t size); Result SetHeapSize(VAddr* out, size_t size); - ResultVal<VAddr> AllocateAndMapMemory(size_t needed_num_pages, size_t align, bool is_map_only, - VAddr region_start, size_t region_num_pages, - KMemoryState state, KMemoryPermission perm, - PAddr map_addr = 0); - Result LockForMapDeviceAddressSpace(bool* out_is_io, VAddr address, size_t size, KMemoryPermission perm, bool is_aligned, bool check_heap); Result LockForUnmapDeviceAddressSpace(VAddr address, size_t size, bool check_heap); @@ -107,8 +116,46 @@ public: return *m_page_table_impl; } + KBlockInfoManager* GetBlockInfoManager() { + return m_block_info_manager; + } + bool CanContain(VAddr addr, size_t size, KMemoryState state) const; + Result MapPages(KProcessAddress* out_addr, size_t num_pages, size_t alignment, + KPhysicalAddress phys_addr, KProcessAddress region_start, + size_t region_num_pages, KMemoryState state, KMemoryPermission perm) { + R_RETURN(this->MapPages(out_addr, num_pages, alignment, phys_addr, true, region_start, + region_num_pages, state, perm)); + } + + Result MapPages(KProcessAddress* out_addr, size_t num_pages, size_t alignment, + KPhysicalAddress phys_addr, KMemoryState state, KMemoryPermission perm) { + R_RETURN(this->MapPages(out_addr, num_pages, alignment, phys_addr, true, + this->GetRegionAddress(state), + this->GetRegionSize(state) / PageSize, state, perm)); + } + + Result MapPages(KProcessAddress* out_addr, size_t num_pages, KMemoryState state, + KMemoryPermission perm) { + R_RETURN(this->MapPages(out_addr, num_pages, PageSize, 0, false, + this->GetRegionAddress(state), + this->GetRegionSize(state) / PageSize, state, perm)); + } + + Result MapPages(KProcessAddress address, size_t num_pages, KMemoryState state, + KMemoryPermission perm); + Result UnmapPages(KProcessAddress address, size_t num_pages, KMemoryState state); + + Result MapPageGroup(KProcessAddress* out_addr, const KPageGroup& pg, + KProcessAddress region_start, size_t region_num_pages, KMemoryState state, + KMemoryPermission perm); + Result MapPageGroup(KProcessAddress address, const KPageGroup& pg, KMemoryState state, + KMemoryPermission perm); + Result UnmapPageGroup(KProcessAddress address, const KPageGroup& pg, KMemoryState state); + void RemapPageGroup(PageLinkedList* page_list, KProcessAddress address, size_t size, + const KPageGroup& pg); + protected: struct PageLinkedList { private: @@ -162,11 +209,9 @@ private: static constexpr KMemoryAttribute DefaultMemoryIgnoreAttr = KMemoryAttribute::IpcLocked | KMemoryAttribute::DeviceShared; - Result MapPages(VAddr addr, const KPageGroup& page_linked_list, KMemoryPermission perm); - Result MapPages(VAddr* out_addr, size_t num_pages, size_t alignment, PAddr phys_addr, - bool is_pa_valid, VAddr region_start, size_t region_num_pages, - KMemoryState state, KMemoryPermission perm); - Result UnmapPages(VAddr addr, const KPageGroup& page_linked_list); + Result MapPages(KProcessAddress* out_addr, size_t num_pages, size_t alignment, + KPhysicalAddress phys_addr, bool is_pa_valid, KProcessAddress region_start, + size_t region_num_pages, KMemoryState state, KMemoryPermission perm); bool IsRegionContiguous(VAddr addr, u64 size) const; void AddRegionToPages(VAddr start, size_t num_pages, KPageGroup& page_linked_list); KMemoryInfo QueryInfoImpl(VAddr addr); @@ -261,9 +306,10 @@ private: void CleanupForIpcClientOnServerSetupFailure(PageLinkedList* page_list, VAddr address, size_t size, KMemoryPermission prot_perm); - // HACK: These will be removed once we automatically manage page reference counts. - void HACK_OpenPages(PAddr phys_addr, size_t num_pages); - void HACK_ClosePages(VAddr virt_addr, size_t num_pages); + Result AllocateAndMapPagesImpl(PageLinkedList* page_list, KProcessAddress address, + size_t num_pages, KMemoryPermission perm); + Result MapPageGroupImpl(PageLinkedList* page_list, KProcessAddress address, + const KPageGroup& pg, const KPageProperties properties, bool reuse_ll); mutable KLightLock m_general_lock; mutable KLightLock m_map_physical_memory_lock; @@ -488,6 +534,7 @@ private: std::unique_ptr<Common::PageTable> m_page_table_impl; Core::System& m_system; + KernelCore& m_kernel; }; } // namespace Kernel diff --git a/src/core/hle/kernel/k_priority_queue.h b/src/core/hle/kernel/k_priority_queue.h index cb2512b0b..645c5b531 100644 --- a/src/core/hle/kernel/k_priority_queue.h +++ b/src/core/hle/kernel/k_priority_queue.h @@ -17,35 +17,41 @@ namespace Kernel { class KThread; template <typename T> -concept KPriorityQueueAffinityMask = !std::is_reference_v<T> && requires(T & t) { - { t.GetAffinityMask() } -> Common::ConvertibleTo<u64>; - {t.SetAffinityMask(0)}; +concept KPriorityQueueAffinityMask = ! +std::is_reference_v<T>&& requires(T& t) { + { t.GetAffinityMask() } -> Common::ConvertibleTo<u64>; + { t.SetAffinityMask(0) }; - { t.GetAffinity(0) } -> std::same_as<bool>; - {t.SetAffinity(0, false)}; - {t.SetAll()}; -}; + { t.GetAffinity(0) } -> std::same_as<bool>; + { t.SetAffinity(0, false) }; + { t.SetAll() }; + }; template <typename T> -concept KPriorityQueueMember = !std::is_reference_v<T> && requires(T & t) { - {typename T::QueueEntry()}; - {(typename T::QueueEntry()).Initialize()}; - {(typename T::QueueEntry()).SetPrev(std::addressof(t))}; - {(typename T::QueueEntry()).SetNext(std::addressof(t))}; - { (typename T::QueueEntry()).GetNext() } -> std::same_as<T*>; - { (typename T::QueueEntry()).GetPrev() } -> std::same_as<T*>; - { t.GetPriorityQueueEntry(0) } -> std::same_as<typename T::QueueEntry&>; - - {t.GetAffinityMask()}; - { std::remove_cvref_t<decltype(t.GetAffinityMask())>() } -> KPriorityQueueAffinityMask; - - { t.GetActiveCore() } -> Common::ConvertibleTo<s32>; - { t.GetPriority() } -> Common::ConvertibleTo<s32>; - { t.IsDummyThread() } -> Common::ConvertibleTo<bool>; -}; +concept KPriorityQueueMember = ! +std::is_reference_v<T>&& requires(T& t) { + { typename T::QueueEntry() }; + { (typename T::QueueEntry()).Initialize() }; + { (typename T::QueueEntry()).SetPrev(std::addressof(t)) }; + { (typename T::QueueEntry()).SetNext(std::addressof(t)) }; + { (typename T::QueueEntry()).GetNext() } -> std::same_as<T*>; + { (typename T::QueueEntry()).GetPrev() } -> std::same_as<T*>; + { + t.GetPriorityQueueEntry(0) + } -> std::same_as<typename T::QueueEntry&>; + + { t.GetAffinityMask() }; + { + std::remove_cvref_t<decltype(t.GetAffinityMask())>() + } -> KPriorityQueueAffinityMask; + + { t.GetActiveCore() } -> Common::ConvertibleTo<s32>; + { t.GetPriority() } -> Common::ConvertibleTo<s32>; + { t.IsDummyThread() } -> Common::ConvertibleTo<bool>; + }; template <typename Member, size_t NumCores_, int LowestPriority, int HighestPriority> -requires KPriorityQueueMember<Member> + requires KPriorityQueueMember<Member> class KPriorityQueue { public: using AffinityMaskType = std::remove_cv_t< diff --git a/src/core/hle/kernel/k_process.cpp b/src/core/hle/kernel/k_process.cpp index a1abf5d68..e201bb0cd 100644 --- a/src/core/hle/kernel/k_process.cpp +++ b/src/core/hle/kernel/k_process.cpp @@ -417,9 +417,8 @@ Result KProcess::LoadFromMetadata(const FileSys::ProgramMetadata& metadata, std: } void KProcess::Run(s32 main_thread_priority, u64 stack_size) { - AllocateMainThreadStack(stack_size); + ASSERT(AllocateMainThreadStack(stack_size) == ResultSuccess); resource_limit->Reserve(LimitableResource::ThreadCountMax, 1); - resource_limit->Reserve(LimitableResource::PhysicalMemoryMax, main_thread_stack_size); const std::size_t heap_capacity{memory_usage_capacity - (main_thread_stack_size + image_size)}; ASSERT(!page_table.SetMaxHeapSize(heap_capacity).IsError()); @@ -675,20 +674,31 @@ void KProcess::ChangeState(State new_state) { } Result KProcess::AllocateMainThreadStack(std::size_t stack_size) { - ASSERT(stack_size); - - // The kernel always ensures that the given stack size is page aligned. - main_thread_stack_size = Common::AlignUp(stack_size, PageSize); - - const VAddr start{page_table.GetStackRegionStart()}; - const std::size_t size{page_table.GetStackRegionEnd() - start}; - - CASCADE_RESULT(main_thread_stack_top, - page_table.AllocateAndMapMemory( - main_thread_stack_size / PageSize, PageSize, false, start, size / PageSize, - KMemoryState::Stack, KMemoryPermission::UserReadWrite)); + // Ensure that we haven't already allocated stack. + ASSERT(main_thread_stack_size == 0); + + // Ensure that we're allocating a valid stack. + stack_size = Common::AlignUp(stack_size, PageSize); + // R_UNLESS(stack_size + image_size <= m_max_process_memory, ResultOutOfMemory); + R_UNLESS(stack_size + image_size >= image_size, ResultOutOfMemory); + + // Place a tentative reservation of memory for our new stack. + KScopedResourceReservation mem_reservation(this, Svc::LimitableResource::PhysicalMemoryMax, + stack_size); + R_UNLESS(mem_reservation.Succeeded(), ResultLimitReached); + + // Allocate and map our stack. + if (stack_size) { + KProcessAddress stack_bottom; + R_TRY(page_table.MapPages(std::addressof(stack_bottom), stack_size / PageSize, + KMemoryState::Stack, KMemoryPermission::UserReadWrite)); + + main_thread_stack_top = stack_bottom + stack_size; + main_thread_stack_size = stack_size; + } - main_thread_stack_top += main_thread_stack_size; + // We succeeded! Commit our memory reservation. + mem_reservation.Commit(); R_SUCCEED(); } diff --git a/src/core/hle/kernel/k_scoped_lock.h b/src/core/hle/kernel/k_scoped_lock.h index 857e21156..59b3e32ae 100644 --- a/src/core/hle/kernel/k_scoped_lock.h +++ b/src/core/hle/kernel/k_scoped_lock.h @@ -9,13 +9,14 @@ namespace Kernel { template <typename T> -concept KLockable = !std::is_reference_v<T> && requires(T & t) { - { t.Lock() } -> std::same_as<void>; - { t.Unlock() } -> std::same_as<void>; -}; +concept KLockable = ! +std::is_reference_v<T>&& requires(T& t) { + { t.Lock() } -> std::same_as<void>; + { t.Unlock() } -> std::same_as<void>; + }; template <typename T> -requires KLockable<T> + requires KLockable<T> class [[nodiscard]] KScopedLock { public: explicit KScopedLock(T* l) : lock_ptr(l) { diff --git a/src/core/hle/kernel/k_shared_memory.cpp b/src/core/hle/kernel/k_shared_memory.cpp index 0aa68103c..df505edfe 100644 --- a/src/core/hle/kernel/k_shared_memory.cpp +++ b/src/core/hle/kernel/k_shared_memory.cpp @@ -13,10 +13,7 @@ namespace Kernel { KSharedMemory::KSharedMemory(KernelCore& kernel_) : KAutoObjectWithSlabHeapAndContainer{kernel_} {} - -KSharedMemory::~KSharedMemory() { - kernel.GetSystemResourceLimit()->Release(LimitableResource::PhysicalMemoryMax, size); -} +KSharedMemory::~KSharedMemory() = default; Result KSharedMemory::Initialize(Core::DeviceMemory& device_memory_, KProcess* owner_process_, Svc::MemoryPermission owner_permission_, @@ -49,7 +46,8 @@ Result KSharedMemory::Initialize(Core::DeviceMemory& device_memory_, KProcess* o R_UNLESS(physical_address != 0, ResultOutOfMemory); //! Insert the result into our page group. - page_group.emplace(physical_address, num_pages); + page_group.emplace(kernel, &kernel.GetSystemSystemResource().GetBlockInfoManager()); + page_group->AddBlock(physical_address, num_pages); // Commit our reservation. memory_reservation.Commit(); @@ -62,7 +60,7 @@ Result KSharedMemory::Initialize(Core::DeviceMemory& device_memory_, KProcess* o is_initialized = true; // Clear all pages in the memory. - for (const auto& block : page_group->Nodes()) { + for (const auto& block : *page_group) { std::memset(device_memory_.GetPointer<void>(block.GetAddress()), 0, block.GetSize()); } @@ -71,13 +69,8 @@ Result KSharedMemory::Initialize(Core::DeviceMemory& device_memory_, KProcess* o void KSharedMemory::Finalize() { // Close and finalize the page group. - // page_group->Close(); - // page_group->Finalize(); - - //! HACK: Manually close. - for (const auto& block : page_group->Nodes()) { - kernel.MemoryManager().Close(block.GetAddress(), block.GetNumPages()); - } + page_group->Close(); + page_group->Finalize(); // Release the memory reservation. resource_limit->Release(LimitableResource::PhysicalMemoryMax, size); @@ -101,15 +94,15 @@ Result KSharedMemory::Map(KProcess& target_process, VAddr address, std::size_t m R_UNLESS(map_perm == test_perm, ResultInvalidNewMemoryPermission); } - return target_process.PageTable().MapPages(address, *page_group, KMemoryState::Shared, - ConvertToKMemoryPermission(map_perm)); + return target_process.PageTable().MapPageGroup(address, *page_group, KMemoryState::Shared, + ConvertToKMemoryPermission(map_perm)); } Result KSharedMemory::Unmap(KProcess& target_process, VAddr address, std::size_t unmap_size) { // Validate the size. R_UNLESS(size == unmap_size, ResultInvalidSize); - return target_process.PageTable().UnmapPages(address, *page_group, KMemoryState::Shared); + return target_process.PageTable().UnmapPageGroup(address, *page_group, KMemoryState::Shared); } } // namespace Kernel diff --git a/src/core/hle/kernel/k_thread.cpp b/src/core/hle/kernel/k_thread.cpp index 21207fe99..84ff3c64b 100644 --- a/src/core/hle/kernel/k_thread.cpp +++ b/src/core/hle/kernel/k_thread.cpp @@ -330,7 +330,7 @@ void KThread::Finalize() { KThread* const waiter = std::addressof(*it); // The thread shouldn't be a kernel waiter. - ASSERT(!IsKernelAddressKey(waiter->GetAddressKey())); + ASSERT(!waiter->GetAddressKeyIsKernel()); // Clear the lock owner. waiter->SetLockOwner(nullptr); @@ -763,19 +763,6 @@ void KThread::Continue() { KScheduler::OnThreadStateChanged(kernel, this, old_state); } -void KThread::WaitUntilSuspended() { - // Make sure we have a suspend requested. - ASSERT(IsSuspendRequested()); - - // Loop until the thread is not executing on any core. - for (std::size_t i = 0; i < static_cast<std::size_t>(Core::Hardware::NUM_CPU_CORES); ++i) { - KThread* core_thread{}; - do { - core_thread = kernel.Scheduler(i).GetSchedulerCurrentThread(); - } while (core_thread == this); - } -} - Result KThread::SetActivity(Svc::ThreadActivity activity) { // Lock ourselves. KScopedLightLock lk(activity_pause_lock); @@ -897,7 +884,7 @@ void KThread::AddWaiterImpl(KThread* thread) { } // Keep track of how many kernel waiters we have. - if (IsKernelAddressKey(thread->GetAddressKey())) { + if (thread->GetAddressKeyIsKernel()) { ASSERT((num_kernel_waiters++) >= 0); KScheduler::SetSchedulerUpdateNeeded(kernel); } @@ -911,7 +898,7 @@ void KThread::RemoveWaiterImpl(KThread* thread) { ASSERT(kernel.GlobalSchedulerContext().IsLocked()); // Keep track of how many kernel waiters we have. - if (IsKernelAddressKey(thread->GetAddressKey())) { + if (thread->GetAddressKeyIsKernel()) { ASSERT((num_kernel_waiters--) > 0); KScheduler::SetSchedulerUpdateNeeded(kernel); } @@ -987,7 +974,7 @@ KThread* KThread::RemoveWaiterByKey(s32* out_num_waiters, VAddr key) { KThread* thread = std::addressof(*it); // Keep track of how many kernel waiters we have. - if (IsKernelAddressKey(thread->GetAddressKey())) { + if (thread->GetAddressKeyIsKernel()) { ASSERT((num_kernel_waiters--) > 0); KScheduler::SetSchedulerUpdateNeeded(kernel); } diff --git a/src/core/hle/kernel/k_thread.h b/src/core/hle/kernel/k_thread.h index 7cd94a340..8b8dc51be 100644 --- a/src/core/hle/kernel/k_thread.h +++ b/src/core/hle/kernel/k_thread.h @@ -214,8 +214,6 @@ public: void Continue(); - void WaitUntilSuspended(); - constexpr void SetSyncedIndex(s32 index) { synced_index = index; } @@ -607,13 +605,30 @@ public: return address_key_value; } - void SetAddressKey(VAddr key) { + [[nodiscard]] bool GetAddressKeyIsKernel() const { + return address_key_is_kernel; + } + + //! NB: intentional deviation from official kernel. + // + // Separate SetAddressKey into user and kernel versions + // to cope with arbitrary host pointers making their way + // into things. + + void SetUserAddressKey(VAddr key) { address_key = key; + address_key_is_kernel = false; } - void SetAddressKey(VAddr key, u32 val) { + void SetUserAddressKey(VAddr key, u32 val) { address_key = key; address_key_value = val; + address_key_is_kernel = false; + } + + void SetKernelAddressKey(VAddr key) { + address_key = key; + address_key_is_kernel = true; } void ClearWaitQueue() { @@ -662,7 +677,7 @@ private: union SyncObjectBuffer { std::array<KSynchronizationObject*, Svc::ArgumentHandleCountMax> sync_objects{}; std::array<Handle, - Svc::ArgumentHandleCountMax*(sizeof(KSynchronizationObject*) / sizeof(Handle))> + Svc::ArgumentHandleCountMax * (sizeof(KSynchronizationObject*) / sizeof(Handle))> handles; constexpr SyncObjectBuffer() {} }; @@ -683,10 +698,8 @@ private: }; template <typename T> - requires( - std::same_as<T, KThread> || - std::same_as<T, RedBlackKeyType>) static constexpr int Compare(const T& lhs, - const KThread& rhs) { + requires(std::same_as<T, KThread> || std::same_as<T, RedBlackKeyType>) + static constexpr int Compare(const T& lhs, const KThread& rhs) { const u64 l_key = lhs.GetConditionVariableKey(); const u64 r_key = rhs.GetConditionVariableKey(); @@ -772,6 +785,7 @@ private: bool debug_attached{}; s8 priority_inheritance_count{}; bool resource_limit_release_hint{}; + bool address_key_is_kernel{}; StackParameters stack_parameters{}; Common::SpinLock context_guard{}; diff --git a/src/core/hle/kernel/k_thread_local_page.h b/src/core/hle/kernel/k_thread_local_page.h index fe0cff084..71254eb55 100644 --- a/src/core/hle/kernel/k_thread_local_page.h +++ b/src/core/hle/kernel/k_thread_local_page.h @@ -70,10 +70,8 @@ public: } template <typename T> - requires(std::same_as<T, KThreadLocalPage> || - std::same_as<T, RedBlackKeyType>) static constexpr int Compare(const T& lhs, - const KThreadLocalPage& - rhs) { + requires(std::same_as<T, KThreadLocalPage> || std::same_as<T, RedBlackKeyType>) + static constexpr int Compare(const T& lhs, const KThreadLocalPage& rhs) { const VAddr lval = GetRedBlackKey(lhs); const VAddr rval = GetRedBlackKey(rhs); diff --git a/src/core/hle/kernel/kernel.cpp b/src/core/hle/kernel/kernel.cpp index 1fb25f221..d9eafe261 100644 --- a/src/core/hle/kernel/kernel.cpp +++ b/src/core/hle/kernel/kernel.cpp @@ -1198,28 +1198,35 @@ void KernelCore::Suspend(bool suspended) { const bool should_suspend{exception_exited || suspended}; const auto activity = should_suspend ? ProcessActivity::Paused : ProcessActivity::Runnable; - std::vector<KScopedAutoObject<KThread>> process_threads; - { - KScopedSchedulerLock sl{*this}; + //! This refers to the application process, not the current process. + KScopedAutoObject<KProcess> process = CurrentProcess(); + if (process.IsNull()) { + return; + } - if (auto* process = CurrentProcess(); process != nullptr) { - process->SetActivity(activity); + // Set the new activity. + process->SetActivity(activity); - if (!should_suspend) { - // Runnable now; no need to wait. - return; - } + // Wait for process execution to stop. + bool must_wait{should_suspend}; + + // KernelCore::Suspend must be called from locked context, or we + // could race another call to SetActivity, interfering with waiting. + while (must_wait) { + KScopedSchedulerLock sl{*this}; + + // Assume that all threads have finished running. + must_wait = false; - for (auto* thread : process->GetThreadList()) { - process_threads.emplace_back(thread); + for (auto i = 0; i < static_cast<s32>(Core::Hardware::NUM_CPU_CORES); ++i) { + if (Scheduler(i).GetSchedulerCurrentThread()->GetOwnerProcess() == + process.GetPointerUnsafe()) { + // A thread has not finished running yet. + // Continue waiting. + must_wait = true; } } } - - // Wait for execution to stop. - for (auto& thread : process_threads) { - thread->WaitUntilSuspended(); - } } void KernelCore::ShutdownCores() { diff --git a/src/core/hle/kernel/memory_types.h b/src/core/hle/kernel/memory_types.h index 3975507bd..92b8b37ac 100644 --- a/src/core/hle/kernel/memory_types.h +++ b/src/core/hle/kernel/memory_types.h @@ -14,4 +14,7 @@ constexpr std::size_t PageSize{1 << PageBits}; using Page = std::array<u8, PageSize>; +using KPhysicalAddress = PAddr; +using KProcessAddress = VAddr; + } // namespace Kernel diff --git a/src/core/hle/kernel/svc.cpp b/src/core/hle/kernel/svc.cpp index 788ee2160..67fa5d71c 100644 --- a/src/core/hle/kernel/svc.cpp +++ b/src/core/hle/kernel/svc.cpp @@ -1485,15 +1485,15 @@ static Result MapProcessMemory(Core::System& system, VAddr dst_address, Handle p ResultInvalidMemoryRegion); // Create a new page group. - KPageGroup pg; + KPageGroup pg{system.Kernel(), dst_pt.GetBlockInfoManager()}; R_TRY(src_pt.MakeAndOpenPageGroup( std::addressof(pg), src_address, size / PageSize, KMemoryState::FlagCanMapProcess, KMemoryState::FlagCanMapProcess, KMemoryPermission::None, KMemoryPermission::None, KMemoryAttribute::All, KMemoryAttribute::None)); // Map the group. - R_TRY(dst_pt.MapPages(dst_address, pg, KMemoryState::SharedCode, - KMemoryPermission::UserReadWrite)); + R_TRY(dst_pt.MapPageGroup(dst_address, pg, KMemoryState::SharedCode, + KMemoryPermission::UserReadWrite)); return ResultSuccess; } |