diff options
Diffstat (limited to '')
| -rw-r--r-- | src/core/hle/kernel/k_page_table.cpp | 944 |
1 files changed, 588 insertions, 356 deletions
diff --git a/src/core/hle/kernel/k_page_table.cpp b/src/core/hle/kernel/k_page_table.cpp index 9c7ac22dc..02b5cada4 100644 --- a/src/core/hle/kernel/k_page_table.cpp +++ b/src/core/hle/kernel/k_page_table.cpp @@ -106,9 +106,10 @@ KPageTable::~KPageTable() = default; Result KPageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_type, bool enable_aslr, bool enable_das_merge, bool from_back, - KMemoryManager::Pool pool, VAddr code_addr, + KMemoryManager::Pool pool, KProcessAddress code_addr, size_t code_size, KSystemResource* system_resource, - KResourceLimit* resource_limit) { + KResourceLimit* resource_limit, + Core::Memory::Memory& memory) { const auto GetSpaceStart = [this](KAddressSpaceInfo::Type type) { return KAddressSpaceInfo::GetAddressSpaceStart(m_address_space_width, type); @@ -117,10 +118,13 @@ Result KPageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_type return KAddressSpaceInfo::GetAddressSpaceSize(m_address_space_width, type); }; + // Set the tracking memory + m_memory = std::addressof(memory); + // Set our width and heap/alias sizes m_address_space_width = GetAddressSpaceWidthFromType(as_type); - const VAddr start = 0; - const VAddr end{1ULL << m_address_space_width}; + const KProcessAddress start = 0; + const KProcessAddress end{1ULL << m_address_space_width}; size_t alias_region_size{GetSpaceSize(KAddressSpaceInfo::Type::Alias)}; size_t heap_region_size{GetSpaceSize(KAddressSpaceInfo::Type::Heap)}; @@ -135,8 +139,8 @@ Result KPageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_type // Set code regions and determine remaining constexpr size_t RegionAlignment{2_MiB}; - VAddr process_code_start{}; - VAddr process_code_end{}; + KProcessAddress process_code_start{}; + KProcessAddress process_code_end{}; size_t stack_region_size{}; size_t kernel_map_region_size{}; @@ -149,8 +153,8 @@ Result KPageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_type m_code_region_end = m_code_region_start + GetSpaceSize(KAddressSpaceInfo::Type::Map39Bit); m_alias_code_region_start = m_code_region_start; m_alias_code_region_end = m_code_region_end; - process_code_start = Common::AlignDown(code_addr, RegionAlignment); - process_code_end = Common::AlignUp(code_addr + code_size, RegionAlignment); + process_code_start = Common::AlignDown(GetInteger(code_addr), RegionAlignment); + process_code_end = Common::AlignUp(GetInteger(code_addr) + code_size, RegionAlignment); } else { stack_region_size = 0; kernel_map_region_size = 0; @@ -178,7 +182,7 @@ Result KPageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_type m_resource_limit = resource_limit; // Determine the region we can place our undetermineds in - VAddr alloc_start{}; + KProcessAddress alloc_start{}; size_t alloc_size{}; if ((process_code_start - m_code_region_start) >= (end - process_code_end)) { alloc_start = m_code_region_start; @@ -292,7 +296,7 @@ Result KPageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_type : KMemoryManager::Direction::FromFront); // Ensure that we regions inside our address space - auto IsInAddressSpace = [&](VAddr addr) { + auto IsInAddressSpace = [&](KProcessAddress addr) { return m_address_space_start <= addr && addr <= m_address_space_end; }; ASSERT(IsInAddressSpace(m_alias_region_start)); @@ -305,14 +309,14 @@ Result KPageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_type ASSERT(IsInAddressSpace(m_kernel_map_region_end)); // Ensure that we selected regions that don't overlap - const VAddr alias_start{m_alias_region_start}; - const VAddr alias_last{m_alias_region_end - 1}; - const VAddr heap_start{m_heap_region_start}; - const VAddr heap_last{m_heap_region_end - 1}; - const VAddr stack_start{m_stack_region_start}; - const VAddr stack_last{m_stack_region_end - 1}; - const VAddr kmap_start{m_kernel_map_region_start}; - const VAddr kmap_last{m_kernel_map_region_end - 1}; + const KProcessAddress alias_start{m_alias_region_start}; + const KProcessAddress alias_last{m_alias_region_end - 1}; + const KProcessAddress heap_start{m_heap_region_start}; + const KProcessAddress heap_last{m_heap_region_end - 1}; + const KProcessAddress stack_start{m_stack_region_start}; + const KProcessAddress stack_last{m_stack_region_end - 1}; + const KProcessAddress kmap_start{m_kernel_map_region_start}; + const KProcessAddress kmap_last{m_kernel_map_region_end - 1}; ASSERT(alias_last < heap_start || heap_last < alias_start); ASSERT(alias_last < stack_start || stack_last < alias_start); ASSERT(alias_last < kmap_start || kmap_last < alias_start); @@ -334,9 +338,10 @@ Result KPageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_type void KPageTable::Finalize() { // Finalize memory blocks. - m_memory_block_manager.Finalize(m_memory_block_slab_manager, [&](VAddr addr, u64 size) { - m_system.Memory().UnmapRegion(*m_page_table_impl, addr, size); - }); + m_memory_block_manager.Finalize(m_memory_block_slab_manager, + [&](KProcessAddress addr, u64 size) { + m_memory->UnmapRegion(*m_page_table_impl, addr, size); + }); // Release any insecure mapped memory. if (m_mapped_insecure_memory) { @@ -352,7 +357,7 @@ void KPageTable::Finalize() { m_page_table_impl.reset(); } -Result KPageTable::MapProcessCode(VAddr addr, size_t num_pages, KMemoryState state, +Result KPageTable::MapProcessCode(KProcessAddress addr, size_t num_pages, KMemoryState state, KMemoryPermission perm) { const u64 size{num_pages * PageSize}; @@ -388,7 +393,8 @@ Result KPageTable::MapProcessCode(VAddr addr, size_t num_pages, KMemoryState sta R_SUCCEED(); } -Result KPageTable::MapCodeMemory(VAddr dst_address, VAddr src_address, size_t size) { +Result KPageTable::MapCodeMemory(KProcessAddress dst_address, KProcessAddress src_address, + size_t size) { // Validate the mapping request. R_UNLESS(this->CanContain(dst_address, size, KMemoryState::AliasCode), ResultInvalidMemoryRegion); @@ -435,6 +441,9 @@ Result KPageTable::MapCodeMemory(VAddr dst_address, VAddr src_address, size_t si KPageGroup pg{m_kernel, m_block_info_manager}; AddRegionToPages(src_address, num_pages, pg); + // We're going to perform an update, so create a helper. + KScopedPageTableUpdater updater(this); + // Reprotect the source as kernel-read/not mapped. const auto new_perm = static_cast<KMemoryPermission>(KMemoryPermission::KernelRead | KMemoryPermission::NotMapped); @@ -447,7 +456,10 @@ Result KPageTable::MapCodeMemory(VAddr dst_address, VAddr src_address, size_t si }); // Map the alias pages. - R_TRY(MapPages(dst_address, pg, new_perm)); + const KPageProperties dst_properties = {new_perm, false, false, + DisableMergeAttribute::DisableHead}; + R_TRY( + this->MapPageGroupImpl(updater.GetPageList(), dst_address, pg, dst_properties, false)); // We successfully mapped the alias pages, so we don't need to unprotect the src pages on // failure. @@ -467,7 +479,8 @@ Result KPageTable::MapCodeMemory(VAddr dst_address, VAddr src_address, size_t si R_SUCCEED(); } -Result KPageTable::UnmapCodeMemory(VAddr dst_address, VAddr src_address, size_t size, +Result KPageTable::UnmapCodeMemory(KProcessAddress dst_address, KProcessAddress src_address, + size_t size, ICacheInvalidationStrategy icache_invalidation_strategy) { // Validate the mapping request. R_UNLESS(this->CanContain(dst_address, size, KMemoryState::AliasCode), @@ -519,7 +532,7 @@ Result KPageTable::UnmapCodeMemory(VAddr dst_address, VAddr src_address, size_t SCOPE_EXIT({ if (reprotected_pages && any_code_pages) { if (icache_invalidation_strategy == ICacheInvalidationStrategy::InvalidateRange) { - m_system.InvalidateCpuInstructionCacheRange(dst_address, size); + m_system.InvalidateCpuInstructionCacheRange(GetInteger(dst_address), size); } else { m_system.InvalidateCpuInstructionCaches(); } @@ -569,9 +582,10 @@ Result KPageTable::UnmapCodeMemory(VAddr dst_address, VAddr src_address, size_t R_SUCCEED(); } -VAddr KPageTable::FindFreeArea(VAddr region_start, size_t region_num_pages, size_t num_pages, - size_t alignment, size_t offset, size_t guard_pages) { - VAddr address = 0; +KProcessAddress KPageTable::FindFreeArea(KProcessAddress region_start, size_t region_num_pages, + size_t num_pages, size_t alignment, size_t offset, + size_t guard_pages) { + KProcessAddress address = 0; if (num_pages <= region_num_pages) { if (this->IsAslrEnabled()) { @@ -587,7 +601,7 @@ VAddr KPageTable::FindFreeArea(VAddr region_start, size_t region_num_pages, size return address; } -Result KPageTable::MakePageGroup(KPageGroup& pg, VAddr addr, size_t num_pages) { +Result KPageTable::MakePageGroup(KPageGroup& pg, KProcessAddress addr, size_t num_pages) { ASSERT(this->IsLockedByCurrentThread()); const size_t size = num_pages * PageSize; @@ -598,11 +612,11 @@ Result KPageTable::MakePageGroup(KPageGroup& pg, VAddr addr, size_t num_pages) { // Begin traversal. Common::PageTable::TraversalContext context; Common::PageTable::TraversalEntry next_entry; - R_UNLESS(m_page_table_impl->BeginTraversal(next_entry, context, addr), + R_UNLESS(m_page_table_impl->BeginTraversal(next_entry, context, GetInteger(addr)), ResultInvalidCurrentMemory); // Prepare tracking variables. - PAddr cur_addr = next_entry.phys_addr; + KPhysicalAddress cur_addr = next_entry.phys_addr; size_t cur_size = next_entry.block_size - (cur_addr & (next_entry.block_size - 1)); size_t tot_size = cur_size; @@ -640,7 +654,7 @@ Result KPageTable::MakePageGroup(KPageGroup& pg, VAddr addr, size_t num_pages) { R_SUCCEED(); } -bool KPageTable::IsValidPageGroup(const KPageGroup& pg, VAddr addr, size_t num_pages) { +bool KPageTable::IsValidPageGroup(const KPageGroup& pg, KProcessAddress addr, size_t num_pages) { ASSERT(this->IsLockedByCurrentThread()); const size_t size = num_pages * PageSize; @@ -653,7 +667,7 @@ bool KPageTable::IsValidPageGroup(const KPageGroup& pg, VAddr addr, size_t num_p // We're going to validate that the group we'd expect is the group we see. auto cur_it = pg.begin(); - PAddr cur_block_address = cur_it->GetAddress(); + KPhysicalAddress cur_block_address = cur_it->GetAddress(); size_t cur_block_pages = cur_it->GetNumPages(); auto UpdateCurrentIterator = [&]() { @@ -671,12 +685,12 @@ bool KPageTable::IsValidPageGroup(const KPageGroup& pg, VAddr addr, size_t num_p // Begin traversal. Common::PageTable::TraversalContext context; Common::PageTable::TraversalEntry next_entry; - if (!m_page_table_impl->BeginTraversal(next_entry, context, addr)) { + if (!m_page_table_impl->BeginTraversal(next_entry, context, GetInteger(addr))) { return false; } // Prepare tracking variables. - PAddr cur_addr = next_entry.phys_addr; + KPhysicalAddress cur_addr = next_entry.phys_addr; size_t cur_size = next_entry.block_size - (cur_addr & (next_entry.block_size - 1)); size_t tot_size = cur_size; @@ -728,8 +742,8 @@ bool KPageTable::IsValidPageGroup(const KPageGroup& pg, VAddr addr, size_t num_p return cur_block_address == cur_addr && cur_block_pages == (cur_size / PageSize); } -Result KPageTable::UnmapProcessMemory(VAddr dst_addr, size_t size, KPageTable& src_page_table, - VAddr src_addr) { +Result KPageTable::UnmapProcessMemory(KProcessAddress dst_addr, size_t size, + KPageTable& src_page_table, KProcessAddress src_addr) { // Acquire the table locks. KScopedLightLockPair lk(src_page_table.m_general_lock, m_general_lock); @@ -768,8 +782,8 @@ Result KPageTable::UnmapProcessMemory(VAddr dst_addr, size_t size, KPageTable& s } Result KPageTable::SetupForIpcClient(PageLinkedList* page_list, size_t* out_blocks_needed, - VAddr address, size_t size, KMemoryPermission test_perm, - KMemoryState dst_state) { + KProcessAddress address, size_t size, + KMemoryPermission test_perm, KMemoryState dst_state) { // Validate pre-conditions. ASSERT(this->IsLockedByCurrentThread()); ASSERT(test_perm == KMemoryPermission::UserReadWrite || @@ -784,10 +798,10 @@ Result KPageTable::SetupForIpcClient(PageLinkedList* page_list, size_t* out_bloc : KMemoryPermission::UserRead; // Get aligned extents. - const VAddr aligned_src_start = Common::AlignDown((address), PageSize); - const VAddr aligned_src_end = Common::AlignUp((address) + size, PageSize); - const VAddr mapping_src_start = Common::AlignUp((address), PageSize); - const VAddr mapping_src_end = Common::AlignDown((address) + size, PageSize); + const KProcessAddress aligned_src_start = Common::AlignDown(GetInteger(address), PageSize); + const KProcessAddress aligned_src_end = Common::AlignUp(GetInteger(address) + size, PageSize); + const KProcessAddress mapping_src_start = Common::AlignUp(GetInteger(address), PageSize); + const KProcessAddress mapping_src_end = Common::AlignDown(GetInteger(address) + size, PageSize); const auto aligned_src_last = (aligned_src_end)-1; const auto mapping_src_last = (mapping_src_end)-1; @@ -834,14 +848,15 @@ Result KPageTable::SetupForIpcClient(PageLinkedList* page_list, size_t* out_bloc test_attr_mask, KMemoryAttribute::None)); if (mapping_src_start < mapping_src_end && (mapping_src_start) < info.GetEndAddress() && - info.GetAddress() < (mapping_src_end)) { - const auto cur_start = - info.GetAddress() >= (mapping_src_start) ? info.GetAddress() : (mapping_src_start); + info.GetAddress() < GetInteger(mapping_src_end)) { + const auto cur_start = info.GetAddress() >= GetInteger(mapping_src_start) + ? info.GetAddress() + : (mapping_src_start); const auto cur_end = mapping_src_last >= info.GetLastAddress() ? info.GetEndAddress() : (mapping_src_end); const size_t cur_size = cur_end - cur_start; - if (info.GetAddress() < (mapping_src_start)) { + if (info.GetAddress() < GetInteger(mapping_src_start)) { ++blocks_needed; } if (mapping_src_last < info.GetLastAddress()) { @@ -876,30 +891,32 @@ Result KPageTable::SetupForIpcClient(PageLinkedList* page_list, size_t* out_bloc R_SUCCEED(); } -Result KPageTable::SetupForIpcServer(VAddr* out_addr, size_t size, VAddr src_addr, - KMemoryPermission test_perm, KMemoryState dst_state, - KPageTable& src_page_table, bool send) { +Result KPageTable::SetupForIpcServer(KProcessAddress* out_addr, size_t size, + KProcessAddress src_addr, KMemoryPermission test_perm, + KMemoryState dst_state, KPageTable& src_page_table, + bool send) { ASSERT(this->IsLockedByCurrentThread()); ASSERT(src_page_table.IsLockedByCurrentThread()); // Check that we can theoretically map. - const VAddr region_start = m_alias_region_start; + const KProcessAddress region_start = m_alias_region_start; const size_t region_size = m_alias_region_end - m_alias_region_start; R_UNLESS(size < region_size, ResultOutOfAddressSpace); // Get aligned source extents. - const VAddr src_start = src_addr; - const VAddr src_end = src_addr + size; - const VAddr aligned_src_start = Common::AlignDown((src_start), PageSize); - const VAddr aligned_src_end = Common::AlignUp((src_start) + size, PageSize); - const VAddr mapping_src_start = Common::AlignUp((src_start), PageSize); - const VAddr mapping_src_end = Common::AlignDown((src_start) + size, PageSize); + const KProcessAddress src_start = src_addr; + const KProcessAddress src_end = src_addr + size; + const KProcessAddress aligned_src_start = Common::AlignDown(GetInteger(src_start), PageSize); + const KProcessAddress aligned_src_end = Common::AlignUp(GetInteger(src_start) + size, PageSize); + const KProcessAddress mapping_src_start = Common::AlignUp(GetInteger(src_start), PageSize); + const KProcessAddress mapping_src_end = + Common::AlignDown(GetInteger(src_start) + size, PageSize); const size_t aligned_src_size = aligned_src_end - aligned_src_start; const size_t mapping_src_size = (mapping_src_start < mapping_src_end) ? (mapping_src_end - mapping_src_start) : 0; // Select a random address to map at. - VAddr dst_addr = + KProcessAddress dst_addr = this->FindFreeArea(region_start, region_size / PageSize, aligned_src_size / PageSize, PageSize, 0, this->GetNumGuardPages()); @@ -924,9 +941,9 @@ Result KPageTable::SetupForIpcServer(VAddr* out_addr, size_t size, VAddr src_add R_UNLESS(memory_reservation.Succeeded(), ResultLimitReached); // Ensure that we manage page references correctly. - PAddr start_partial_page = 0; - PAddr end_partial_page = 0; - VAddr cur_mapped_addr = dst_addr; + KPhysicalAddress start_partial_page = 0; + KPhysicalAddress end_partial_page = 0; + KProcessAddress cur_mapped_addr = dst_addr; // If the partial pages are mapped, an extra reference will have been opened. Otherwise, they'll // free on scope exit. @@ -971,11 +988,12 @@ Result KPageTable::SetupForIpcServer(VAddr* out_addr, size_t size, VAddr src_add // Begin traversal. Common::PageTable::TraversalContext context; Common::PageTable::TraversalEntry next_entry; - bool traverse_valid = src_impl.BeginTraversal(next_entry, context, aligned_src_start); + bool traverse_valid = + src_impl.BeginTraversal(next_entry, context, GetInteger(aligned_src_start)); ASSERT(traverse_valid); // Prepare tracking variables. - PAddr cur_block_addr = next_entry.phys_addr; + KPhysicalAddress cur_block_addr = next_entry.phys_addr; size_t cur_block_size = next_entry.block_size - ((cur_block_addr) & (next_entry.block_size - 1)); size_t tot_block_size = cur_block_size; @@ -983,7 +1001,7 @@ Result KPageTable::SetupForIpcServer(VAddr* out_addr, size_t size, VAddr src_add // Map the start page, if we have one. if (start_partial_page != 0) { // Ensure the page holds correct data. - const VAddr start_partial_virt = + const KVirtualAddress start_partial_virt = GetHeapVirtualAddress(m_system.Kernel().MemoryLayout(), start_partial_page); if (send) { const size_t partial_offset = src_start - aligned_src_start; @@ -996,21 +1014,22 @@ Result KPageTable::SetupForIpcServer(VAddr* out_addr, size_t size, VAddr src_add clear_size = 0; } - std::memset(m_system.Memory().GetPointer<void>(start_partial_virt), fill_val, + std::memset(m_memory->GetPointer<void>(GetInteger(start_partial_virt)), fill_val, partial_offset); std::memcpy( - m_system.Memory().GetPointer<void>(start_partial_virt + partial_offset), - m_system.Memory().GetPointer<void>( - GetHeapVirtualAddress(m_system.Kernel().MemoryLayout(), cur_block_addr) + - partial_offset), + m_memory->GetPointer<void>(GetInteger(start_partial_virt) + partial_offset), + m_memory->GetPointer<void>(GetInteger(GetHeapVirtualAddress( + m_system.Kernel().MemoryLayout(), cur_block_addr)) + + partial_offset), copy_size); if (clear_size > 0) { - std::memset(m_system.Memory().GetPointer<void>(start_partial_virt + partial_offset + - copy_size), + std::memset(m_memory->GetPointer<void>(GetInteger(start_partial_virt) + + partial_offset + copy_size), fill_val, clear_size); } } else { - std::memset(m_system.Memory().GetPointer<void>(start_partial_virt), fill_val, PageSize); + std::memset(m_memory->GetPointer<void>(GetInteger(start_partial_virt)), fill_val, + PageSize); } // Map the page. @@ -1055,7 +1074,8 @@ Result KPageTable::SetupForIpcServer(VAddr* out_addr, size_t size, VAddr src_add } // Handle the last direct-mapped page. - if (const VAddr mapped_block_end = aligned_src_start + tot_block_size - cur_block_size; + if (const KProcessAddress mapped_block_end = + aligned_src_start + tot_block_size - cur_block_size; mapped_block_end < mapping_src_end) { const size_t last_block_size = mapping_src_end - mapped_block_end; @@ -1078,18 +1098,19 @@ Result KPageTable::SetupForIpcServer(VAddr* out_addr, size_t size, VAddr src_add // Map the end page, if we have one. if (end_partial_page != 0) { // Ensure the page holds correct data. - const VAddr end_partial_virt = + const KVirtualAddress end_partial_virt = GetHeapVirtualAddress(m_system.Kernel().MemoryLayout(), end_partial_page); if (send) { const size_t copy_size = src_end - mapping_src_end; - std::memcpy(m_system.Memory().GetPointer<void>(end_partial_virt), - m_system.Memory().GetPointer<void>(GetHeapVirtualAddress( - m_system.Kernel().MemoryLayout(), cur_block_addr)), + std::memcpy(m_memory->GetPointer<void>(GetInteger(end_partial_virt)), + m_memory->GetPointer<void>(GetInteger(GetHeapVirtualAddress( + m_system.Kernel().MemoryLayout(), cur_block_addr))), copy_size); - std::memset(m_system.Memory().GetPointer<void>(end_partial_virt + copy_size), fill_val, - PageSize - copy_size); + std::memset(m_memory->GetPointer<void>(GetInteger(end_partial_virt) + copy_size), + fill_val, PageSize - copy_size); } else { - std::memset(m_system.Memory().GetPointer<void>(end_partial_virt), fill_val, PageSize); + std::memset(m_memory->GetPointer<void>(GetInteger(end_partial_virt)), fill_val, + PageSize); } // Map the page. @@ -1110,7 +1131,7 @@ Result KPageTable::SetupForIpcServer(VAddr* out_addr, size_t size, VAddr src_add R_SUCCEED(); } -Result KPageTable::SetupForIpc(VAddr* out_dst_addr, size_t size, VAddr src_addr, +Result KPageTable::SetupForIpc(KProcessAddress* out_dst_addr, size_t size, KProcessAddress src_addr, KPageTable& src_page_table, KMemoryPermission test_perm, KMemoryState dst_state, bool send) { // For convenience, alias this. @@ -1136,8 +1157,8 @@ Result KPageTable::SetupForIpc(VAddr* out_dst_addr, size_t size, VAddr src_addr, R_TRY(allocator_result); // Get the mapped extents. - const VAddr src_map_start = Common::AlignUp((src_addr), PageSize); - const VAddr src_map_end = Common::AlignDown((src_addr) + size, PageSize); + const KProcessAddress src_map_start = Common::AlignUp(GetInteger(src_addr), PageSize); + const KProcessAddress src_map_end = Common::AlignDown(GetInteger(src_addr) + size, PageSize); const size_t src_map_size = src_map_end - src_map_start; // Ensure that we clean up appropriately if we fail after this. @@ -1166,7 +1187,8 @@ Result KPageTable::SetupForIpc(VAddr* out_dst_addr, size_t size, VAddr src_addr, R_SUCCEED(); } -Result KPageTable::CleanupForIpcServer(VAddr address, size_t size, KMemoryState dst_state) { +Result KPageTable::CleanupForIpcServer(KProcessAddress address, size_t size, + KMemoryState dst_state) { // Validate the address. R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory); @@ -1190,8 +1212,8 @@ Result KPageTable::CleanupForIpcServer(VAddr address, size_t size, KMemoryState KScopedPageTableUpdater updater(this); // Get aligned extents. - const VAddr aligned_start = Common::AlignDown((address), PageSize); - const VAddr aligned_end = Common::AlignUp((address) + size, PageSize); + const KProcessAddress aligned_start = Common::AlignDown(GetInteger(address), PageSize); + const KProcessAddress aligned_end = Common::AlignUp(GetInteger(address) + size, PageSize); const size_t aligned_size = aligned_end - aligned_start; const size_t aligned_num_pages = aligned_size / PageSize; @@ -1205,22 +1227,23 @@ Result KPageTable::CleanupForIpcServer(VAddr address, size_t size, KMemoryState KMemoryBlockDisableMergeAttribute::Normal); // Release from the resource limit as relevant. - const VAddr mapping_start = Common::AlignUp((address), PageSize); - const VAddr mapping_end = Common::AlignDown((address) + size, PageSize); + const KProcessAddress mapping_start = Common::AlignUp(GetInteger(address), PageSize); + const KProcessAddress mapping_end = Common::AlignDown(GetInteger(address) + size, PageSize); const size_t mapping_size = (mapping_start < mapping_end) ? mapping_end - mapping_start : 0; m_resource_limit->Release(LimitableResource::PhysicalMemoryMax, aligned_size - mapping_size); R_SUCCEED(); } -Result KPageTable::CleanupForIpcClient(VAddr address, size_t size, KMemoryState dst_state) { +Result KPageTable::CleanupForIpcClient(KProcessAddress address, size_t size, + KMemoryState dst_state) { // Validate the address. R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory); // Get aligned source extents. - const VAddr mapping_start = Common::AlignUp((address), PageSize); - const VAddr mapping_end = Common::AlignDown((address) + size, PageSize); - const VAddr mapping_last = mapping_end - 1; + const KProcessAddress mapping_start = Common::AlignUp(GetInteger(address), PageSize); + const KProcessAddress mapping_end = Common::AlignDown(GetInteger(address) + size, PageSize); + const KProcessAddress mapping_last = mapping_end - 1; const size_t mapping_size = (mapping_start < mapping_end) ? (mapping_end - mapping_start) : 0; // If nothing was mapped, we're actually done immediately. @@ -1273,7 +1296,7 @@ Result KPageTable::CleanupForIpcClient(VAddr address, size_t size, KMemoryState KMemoryInfo cur_info = start_it->GetMemoryInfo(); // Create tracking variables. - VAddr cur_address = cur_info.GetAddress(); + KProcessAddress cur_address = cur_info.GetAddress(); size_t cur_size = cur_info.GetSize(); bool cur_perm_eq = cur_info.GetPermission() == cur_info.GetOriginalPermission(); bool cur_needs_set_perm = !cur_perm_eq && cur_info.GetIpcLockCount() == 1; @@ -1346,7 +1369,7 @@ Result KPageTable::CleanupForIpcClient(VAddr address, size_t size, KMemoryState .IsSuccess()); // Create tracking variables. - VAddr cur_address = cur_info.GetAddress(); + KProcessAddress cur_address = cur_info.GetAddress(); size_t cur_size = cur_info.GetSize(); bool cur_perm_eq = cur_info.GetPermission() == cur_info.GetOriginalPermission(); bool cur_needs_set_perm = !cur_perm_eq && cur_info.GetIpcLockCount() == 1; @@ -1433,16 +1456,16 @@ Result KPageTable::CleanupForIpcClient(VAddr address, size_t size, KMemoryState } void KPageTable::CleanupForIpcClientOnServerSetupFailure([[maybe_unused]] PageLinkedList* page_list, - VAddr address, size_t size, + KProcessAddress address, size_t size, KMemoryPermission prot_perm) { ASSERT(this->IsLockedByCurrentThread()); - ASSERT(Common::IsAligned(address, PageSize)); + ASSERT(Common::IsAligned(GetInteger(address), PageSize)); ASSERT(Common::IsAligned(size, PageSize)); // Get the mapped extents. - const VAddr src_map_start = address; - const VAddr src_map_end = address + size; - const VAddr src_map_last = src_map_end - 1; + const KProcessAddress src_map_start = address; + const KProcessAddress src_map_end = address + size; + const KProcessAddress src_map_last = src_map_end - 1; // This function is only invoked when there's something to do. ASSERT(src_map_end > src_map_start); @@ -1452,8 +1475,9 @@ void KPageTable::CleanupForIpcClientOnServerSetupFailure([[maybe_unused]] PageLi while (true) { const KMemoryInfo info = it->GetMemoryInfo(); - const auto cur_start = - info.GetAddress() >= src_map_start ? info.GetAddress() : src_map_start; + const auto cur_start = info.GetAddress() >= GetInteger(src_map_start) + ? info.GetAddress() + : GetInteger(src_map_start); const auto cur_end = src_map_last <= info.GetLastAddress() ? src_map_end : info.GetEndAddress(); @@ -1463,7 +1487,7 @@ void KPageTable::CleanupForIpcClientOnServerSetupFailure([[maybe_unused]] PageLi (info.GetIpcLockCount() != 0 && (info.GetOriginalPermission() & KMemoryPermission::IpcLockChangeMask) != prot_perm)) { // Check if we actually need to fix the protections on the block. - if (cur_end == src_map_end || info.GetAddress() <= src_map_start || + if (cur_end == src_map_end || info.GetAddress() <= GetInteger(src_map_start) || (info.GetPermission() & KMemoryPermission::IpcLockChangeMask) != prot_perm) { ASSERT(Operate(cur_start, (cur_end - cur_start) / PageSize, info.GetPermission(), OperationType::ChangePermissions) @@ -1482,15 +1506,15 @@ void KPageTable::CleanupForIpcClientOnServerSetupFailure([[maybe_unused]] PageLi } } -Result KPageTable::MapPhysicalMemory(VAddr address, size_t size) { +Result KPageTable::MapPhysicalMemory(KProcessAddress address, size_t size) { // Lock the physical memory lock. KScopedLightLock phys_lk(m_map_physical_memory_lock); // Calculate the last address for convenience. - const VAddr last_address = address + size - 1; + const KProcessAddress last_address = address + size - 1; // Define iteration variables. - VAddr cur_address; + KProcessAddress cur_address; size_t mapped_size; // The entire mapping process can be retried. @@ -1522,7 +1546,7 @@ Result KPageTable::MapPhysicalMemory(VAddr address, size_t size) { // Track the memory if it's mapped. if (info.GetState() != KMemoryState::Free) { - mapped_size += VAddr(info.GetEndAddress()) - cur_address; + mapped_size += KProcessAddress(info.GetEndAddress()) - cur_address; } // Advance. @@ -1575,7 +1599,7 @@ Result KPageTable::MapPhysicalMemory(VAddr address, size_t size) { const bool is_free = info.GetState() == KMemoryState::Free; if (is_free) { - if (info.GetAddress() < address) { + if (info.GetAddress() < GetInteger(address)) { ++num_allocator_blocks; } if (last_address < info.GetLastAddress()) { @@ -1593,7 +1617,8 @@ Result KPageTable::MapPhysicalMemory(VAddr address, size_t size) { // Track the memory if it's mapped. if (!is_free) { - checked_mapped_size += VAddr(info.GetEndAddress()) - cur_address; + checked_mapped_size += + KProcessAddress(info.GetEndAddress()) - cur_address; } // Advance. @@ -1621,7 +1646,7 @@ Result KPageTable::MapPhysicalMemory(VAddr address, size_t size) { // Prepare to iterate over the memory. auto pg_it = pg.begin(); - PAddr pg_phys_addr = pg_it->GetAddress(); + KPhysicalAddress pg_phys_addr = pg_it->GetAddress(); size_t pg_pages = pg_it->GetNumPages(); // Reset the current tracking address, and make sure we clean up on failure. @@ -1629,7 +1654,7 @@ Result KPageTable::MapPhysicalMemory(VAddr address, size_t size) { cur_address = address; ON_RESULT_FAILURE { if (cur_address > address) { - const VAddr last_unmap_address = cur_address - 1; + const KProcessAddress last_unmap_address = cur_address - 1; // Iterate, unmapping the pages. cur_address = address; @@ -1646,7 +1671,7 @@ Result KPageTable::MapPhysicalMemory(VAddr address, size_t size) { if (info.GetState() == KMemoryState::Free) { // Determine the range to unmap. const size_t cur_pages = - std::min(VAddr(info.GetEndAddress()) - cur_address, + std::min(KProcessAddress(info.GetEndAddress()) - cur_address, last_unmap_address + 1 - cur_address) / PageSize; @@ -1689,9 +1714,10 @@ Result KPageTable::MapPhysicalMemory(VAddr address, size_t size) { // If it's unmapped, we need to map it. if (info.GetState() == KMemoryState::Free) { // Determine the range to map. - size_t map_pages = std::min(VAddr(info.GetEndAddress()) - cur_address, - last_address + 1 - cur_address) / - PageSize; + size_t map_pages = + std::min(KProcessAddress(info.GetEndAddress()) - cur_address, + last_address + 1 - cur_address) / + PageSize; // While we have pages to map, map them. while (map_pages > 0) { @@ -1748,7 +1774,7 @@ Result KPageTable::MapPhysicalMemory(VAddr address, size_t size) { } } -Result KPageTable::UnmapPhysicalMemory(VAddr address, size_t size) { +Result KPageTable::UnmapPhysicalMemory(KProcessAddress address, size_t size) { // Lock the physical memory lock. KScopedLightLock phys_lk(m_map_physical_memory_lock); @@ -1756,13 +1782,13 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, size_t size) { KScopedLightLock lk(m_general_lock); // Calculate the last address for convenience. - const VAddr last_address = address + size - 1; + const KProcessAddress last_address = address + size - 1; // Define iteration variables. - VAddr map_start_address = 0; - VAddr map_last_address = 0; + KProcessAddress map_start_address = 0; + KProcessAddress map_last_address = 0; - VAddr cur_address; + KProcessAddress cur_address; size_t mapped_size; size_t num_allocator_blocks = 0; @@ -1795,7 +1821,7 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, size_t size) { map_last_address = (last_address >= info.GetLastAddress()) ? info.GetLastAddress() : last_address; - if (info.GetAddress() < address) { + if (info.GetAddress() < GetInteger(address)) { ++num_allocator_blocks; } if (last_address < info.GetLastAddress()) { @@ -1848,7 +1874,7 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, size_t size) { // If the memory state is normal, we need to unmap it. if (info.GetState() == KMemoryState::Normal) { // Determine the range to unmap. - const size_t cur_pages = std::min(VAddr(info.GetEndAddress()) - cur_address, + const size_t cur_pages = std::min(KProcessAddress(info.GetEndAddress()) - cur_address, last_address + 1 - cur_address) / PageSize; @@ -1881,7 +1907,8 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, size_t size) { R_SUCCEED(); } -Result KPageTable::MapMemory(VAddr dst_address, VAddr src_address, size_t size) { +Result KPageTable::MapMemory(KProcessAddress dst_address, KProcessAddress src_address, + size_t size) { // Lock the table. KScopedLightLock lk(m_general_lock); @@ -1902,53 +1929,73 @@ Result KPageTable::MapMemory(VAddr dst_address, VAddr src_address, size_t size) KMemoryAttribute::None)); // Create an update allocator for the source. - Result src_allocator_result{ResultSuccess}; + Result src_allocator_result; KMemoryBlockManagerUpdateAllocator src_allocator(std::addressof(src_allocator_result), m_memory_block_slab_manager, num_src_allocator_blocks); R_TRY(src_allocator_result); // Create an update allocator for the destination. - Result dst_allocator_result{ResultSuccess}; + Result dst_allocator_result; KMemoryBlockManagerUpdateAllocator dst_allocator(std::addressof(dst_allocator_result), m_memory_block_slab_manager, num_dst_allocator_blocks); R_TRY(dst_allocator_result); // Map the memory. - KPageGroup page_linked_list{m_kernel, m_block_info_manager}; - const size_t num_pages{size / PageSize}; - const KMemoryPermission new_src_perm = static_cast<KMemoryPermission>( - KMemoryPermission::KernelRead | KMemoryPermission::NotMapped); - const KMemoryAttribute new_src_attr = KMemoryAttribute::Locked; - - AddRegionToPages(src_address, num_pages, page_linked_list); { + // Determine the number of pages being operated on. + const size_t num_pages = size / PageSize; + + // Create page groups for the memory being unmapped. + KPageGroup pg{m_kernel, m_block_info_manager}; + + // Create the page group representing the source. + R_TRY(this->MakePageGroup(pg, src_address, num_pages)); + + // We're going to perform an update, so create a helper. + KScopedPageTableUpdater updater(this); + // Reprotect the source as kernel-read/not mapped. - auto block_guard = detail::ScopeExit([&] { - Operate(src_address, num_pages, KMemoryPermission::UserReadWrite, - OperationType::ChangePermissions); - }); - R_TRY(Operate(src_address, num_pages, new_src_perm, OperationType::ChangePermissions)); - R_TRY(MapPages(dst_address, page_linked_list, KMemoryPermission::UserReadWrite)); + const KMemoryPermission new_src_perm = static_cast<KMemoryPermission>( + KMemoryPermission::KernelRead | KMemoryPermission::NotMapped); + const KMemoryAttribute new_src_attr = KMemoryAttribute::Locked; + const KPageProperties src_properties = {new_src_perm, false, false, + DisableMergeAttribute::DisableHeadBodyTail}; + R_TRY(this->Operate(src_address, num_pages, src_properties.perm, + OperationType::ChangePermissions)); - block_guard.Cancel(); - } + // Ensure that we unprotect the source pages on failure. + ON_RESULT_FAILURE { + const KPageProperties unprotect_properties = { + KMemoryPermission::UserReadWrite, false, false, + DisableMergeAttribute::EnableHeadBodyTail}; + ASSERT(this->Operate(src_address, num_pages, unprotect_properties.perm, + OperationType::ChangePermissions) == ResultSuccess); + }; - // Apply the memory block updates. - m_memory_block_manager.Update(std::addressof(src_allocator), src_address, num_pages, src_state, - new_src_perm, new_src_attr, - KMemoryBlockDisableMergeAttribute::Locked, - KMemoryBlockDisableMergeAttribute::None); - m_memory_block_manager.Update(std::addressof(dst_allocator), dst_address, num_pages, - KMemoryState::Stack, KMemoryPermission::UserReadWrite, - KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, - KMemoryBlockDisableMergeAttribute::None); + // Map the alias pages. + const KPageProperties dst_map_properties = {KMemoryPermission::UserReadWrite, false, false, + DisableMergeAttribute::DisableHead}; + R_TRY(this->MapPageGroupImpl(updater.GetPageList(), dst_address, pg, dst_map_properties, + false)); + + // Apply the memory block updates. + m_memory_block_manager.Update(std::addressof(src_allocator), src_address, num_pages, + src_state, new_src_perm, new_src_attr, + KMemoryBlockDisableMergeAttribute::Locked, + KMemoryBlockDisableMergeAttribute::None); + m_memory_block_manager.Update( + std::addressof(dst_allocator), dst_address, num_pages, KMemoryState::Stack, + KMemoryPermission::UserReadWrite, KMemoryAttribute::None, + KMemoryBlockDisableMergeAttribute::Normal, KMemoryBlockDisableMergeAttribute::None); + } R_SUCCEED(); } -Result KPageTable::UnmapMemory(VAddr dst_address, VAddr src_address, size_t size) { +Result KPageTable::UnmapMemory(KProcessAddress dst_address, KProcessAddress src_address, + size_t size) { // Lock the table. KScopedLightLock lk(m_general_lock); @@ -1970,108 +2017,209 @@ Result KPageTable::UnmapMemory(VAddr dst_address, VAddr src_address, size_t size KMemoryPermission::None, KMemoryAttribute::All, KMemoryAttribute::None)); // Create an update allocator for the source. - Result src_allocator_result{ResultSuccess}; + Result src_allocator_result; KMemoryBlockManagerUpdateAllocator src_allocator(std::addressof(src_allocator_result), m_memory_block_slab_manager, num_src_allocator_blocks); R_TRY(src_allocator_result); // Create an update allocator for the destination. - Result dst_allocator_result{ResultSuccess}; + Result dst_allocator_result; KMemoryBlockManagerUpdateAllocator dst_allocator(std::addressof(dst_allocator_result), m_memory_block_slab_manager, num_dst_allocator_blocks); R_TRY(dst_allocator_result); - KPageGroup src_pages{m_kernel, m_block_info_manager}; - KPageGroup dst_pages{m_kernel, m_block_info_manager}; - const size_t num_pages{size / PageSize}; + // Unmap the memory. + { + // Determine the number of pages being operated on. + const size_t num_pages = size / PageSize; - AddRegionToPages(src_address, num_pages, src_pages); - AddRegionToPages(dst_address, num_pages, dst_pages); + // Create page groups for the memory being unmapped. + KPageGroup pg{m_kernel, m_block_info_manager}; - R_UNLESS(dst_pages.IsEquivalentTo(src_pages), ResultInvalidMemoryRegion); + // Create the page group representing the destination. + R_TRY(this->MakePageGroup(pg, dst_address, num_pages)); - { - auto block_guard = detail::ScopeExit([&] { MapPages(dst_address, dst_pages, dst_perm); }); + // Ensure the page group is the valid for the source. + R_UNLESS(this->IsValidPageGroup(pg, src_address, num_pages), ResultInvalidMemoryRegion); - R_TRY(Operate(dst_address, num_pages, KMemoryPermission::None, OperationType::Unmap)); - R_TRY(Operate(src_address, num_pages, KMemoryPermission::UserReadWrite, - OperationType::ChangePermissions)); + // We're going to perform an update, so create a helper. + KScopedPageTableUpdater updater(this); - block_guard.Cancel(); - } + // Unmap the aliased copy of the pages. + const KPageProperties dst_unmap_properties = {KMemoryPermission::None, false, false, + DisableMergeAttribute::None}; + R_TRY( + this->Operate(dst_address, num_pages, dst_unmap_properties.perm, OperationType::Unmap)); + + // Ensure that we re-map the aliased pages on failure. + ON_RESULT_FAILURE { + this->RemapPageGroup(updater.GetPageList(), dst_address, size, pg); + }; - // Apply the memory block updates. - m_memory_block_manager.Update(std::addressof(src_allocator), src_address, num_pages, src_state, - KMemoryPermission::UserReadWrite, KMemoryAttribute::None, - KMemoryBlockDisableMergeAttribute::None, - KMemoryBlockDisableMergeAttribute::Locked); - m_memory_block_manager.Update(std::addressof(dst_allocator), dst_address, num_pages, - KMemoryState::None, KMemoryPermission::None, - KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::None, - KMemoryBlockDisableMergeAttribute::Normal); + // Try to set the permissions for the source pages back to what they should be. + const KPageProperties src_properties = {KMemoryPermission::UserReadWrite, false, false, + DisableMergeAttribute::EnableAndMergeHeadBodyTail}; + R_TRY(this->Operate(src_address, num_pages, src_properties.perm, + OperationType::ChangePermissions)); + + // Apply the memory block updates. + m_memory_block_manager.Update( + std::addressof(src_allocator), src_address, num_pages, src_state, + KMemoryPermission::UserReadWrite, KMemoryAttribute::None, + KMemoryBlockDisableMergeAttribute::None, KMemoryBlockDisableMergeAttribute::Locked); + m_memory_block_manager.Update( + std::addressof(dst_allocator), dst_address, num_pages, KMemoryState::None, + KMemoryPermission::None, KMemoryAttribute::None, + KMemoryBlockDisableMergeAttribute::None, KMemoryBlockDisableMergeAttribute::Normal); + } R_SUCCEED(); } -Result KPageTable::MapPages(VAddr addr, const KPageGroup& page_linked_list, - KMemoryPermission perm) { +Result KPageTable::AllocateAndMapPagesImpl(PageLinkedList* page_list, KProcessAddress address, + size_t num_pages, KMemoryPermission perm) { ASSERT(this->IsLockedByCurrentThread()); - VAddr cur_addr{addr}; + // Create a page group to hold the pages we allocate. + KPageGroup pg{m_kernel, m_block_info_manager}; - for (const auto& node : page_linked_list) { - if (const auto result{ - Operate(cur_addr, node.GetNumPages(), perm, OperationType::Map, node.GetAddress())}; - result.IsError()) { - const size_t num_pages{(addr - cur_addr) / PageSize}; + // Allocate the pages. + R_TRY( + m_kernel.MemoryManager().AllocateAndOpen(std::addressof(pg), num_pages, m_allocate_option)); - ASSERT(Operate(addr, num_pages, KMemoryPermission::None, OperationType::Unmap) - .IsSuccess()); + // Ensure that the page group is closed when we're done working with it. + SCOPE_EXIT({ pg.Close(); }); - R_RETURN(result); + // Clear all pages. + for (const auto& it : pg) { + std::memset(m_system.DeviceMemory().GetPointer<void>(it.GetAddress()), m_heap_fill_value, + it.GetSize()); + } + + // Map the pages. + R_RETURN(this->Operate(address, num_pages, pg, OperationType::MapGroup)); +} + +Result KPageTable::MapPageGroupImpl(PageLinkedList* page_list, KProcessAddress address, + const KPageGroup& pg, const KPageProperties properties, + bool reuse_ll) { + ASSERT(this->IsLockedByCurrentThread()); + + // Note the current address, so that we can iterate. + const KProcessAddress start_address = address; + KProcessAddress cur_address = address; + + // Ensure that we clean up on failure. + ON_RESULT_FAILURE { + ASSERT(!reuse_ll); + if (cur_address != start_address) { + const KPageProperties unmap_properties = {KMemoryPermission::None, false, false, + DisableMergeAttribute::None}; + ASSERT(this->Operate(start_address, (cur_address - start_address) / PageSize, + unmap_properties.perm, OperationType::Unmap) == ResultSuccess); } + }; - cur_addr += node.GetNumPages() * PageSize; + // Iterate, mapping all pages in the group. + for (const auto& block : pg) { + // Map and advance. + const KPageProperties cur_properties = + (cur_address == start_address) + ? properties + : KPageProperties{properties.perm, properties.io, properties.uncached, + DisableMergeAttribute::None}; + this->Operate(cur_address, block.GetNumPages(), cur_properties.perm, OperationType::Map, + block.GetAddress()); + cur_address += block.GetSize(); } + // We succeeded! R_SUCCEED(); } -Result KPageTable::MapPages(VAddr address, KPageGroup& page_linked_list, KMemoryState state, - KMemoryPermission perm) { - // Check that the map is in range. - const size_t num_pages{page_linked_list.GetNumPages()}; - const size_t size{num_pages * PageSize}; - R_UNLESS(this->CanContain(address, size, state), ResultInvalidCurrentMemory); +void KPageTable::RemapPageGroup(PageLinkedList* page_list, KProcessAddress address, size_t size, + const KPageGroup& pg) { + ASSERT(this->IsLockedByCurrentThread()); - // Lock the table. - KScopedLightLock lk(m_general_lock); + // Note the current address, so that we can iterate. + const KProcessAddress start_address = address; + const KProcessAddress last_address = start_address + size - 1; + const KProcessAddress end_address = last_address + 1; - // Check the memory state. - R_TRY(this->CheckMemoryState(address, size, KMemoryState::All, KMemoryState::Free, - KMemoryPermission::None, KMemoryPermission::None, - KMemoryAttribute::None, KMemoryAttribute::None)); + // Iterate over the memory. + auto pg_it = pg.begin(); + ASSERT(pg_it != pg.end()); - // Create an update allocator. - Result allocator_result{ResultSuccess}; - KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), - m_memory_block_slab_manager); + KPhysicalAddress pg_phys_addr = pg_it->GetAddress(); + size_t pg_pages = pg_it->GetNumPages(); - // Map the pages. - R_TRY(MapPages(address, page_linked_list, perm)); + auto it = m_memory_block_manager.FindIterator(start_address); + while (true) { + // Check that the iterator is valid. + ASSERT(it != m_memory_block_manager.end()); - // Update the blocks. - m_memory_block_manager.Update(std::addressof(allocator), address, num_pages, state, perm, - KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, - KMemoryBlockDisableMergeAttribute::None); + // Get the memory info. + const KMemoryInfo info = it->GetMemoryInfo(); - R_SUCCEED(); + // Determine the range to map. + KProcessAddress map_address = std::max<KProcessAddress>(info.GetAddress(), start_address); + const KProcessAddress map_end_address = + std::min<KProcessAddress>(info.GetEndAddress(), end_address); + ASSERT(map_end_address != map_address); + + // Determine if we should disable head merge. + const bool disable_head_merge = + info.GetAddress() >= GetInteger(start_address) && + True(info.GetDisableMergeAttribute() & KMemoryBlockDisableMergeAttribute::Normal); + const KPageProperties map_properties = { + info.GetPermission(), false, false, + disable_head_merge ? DisableMergeAttribute::DisableHead : DisableMergeAttribute::None}; + + // While we have pages to map, map them. + size_t map_pages = (map_end_address - map_address) / PageSize; + while (map_pages > 0) { + // Check if we're at the end of the physical block. + if (pg_pages == 0) { + // Ensure there are more pages to map. + ASSERT(pg_it != pg.end()); + + // Advance our physical block. + ++pg_it; + pg_phys_addr = pg_it->GetAddress(); + pg_pages = pg_it->GetNumPages(); + } + + // Map whatever we can. + const size_t cur_pages = std::min(pg_pages, map_pages); + ASSERT(this->Operate(map_address, map_pages, map_properties.perm, OperationType::Map, + pg_phys_addr) == ResultSuccess); + + // Advance. + map_address += cur_pages * PageSize; + map_pages -= cur_pages; + + pg_phys_addr += cur_pages * PageSize; + pg_pages -= cur_pages; + } + + // Check if we're done. + if (last_address <= info.GetLastAddress()) { + break; + } + + // Advance. + ++it; + } + + // Check that we re-mapped precisely the page group. + ASSERT((++pg_it) == pg.end()); } -Result KPageTable::MapPages(VAddr* out_addr, size_t num_pages, size_t alignment, PAddr phys_addr, - bool is_pa_valid, VAddr region_start, size_t region_num_pages, +Result KPageTable::MapPages(KProcessAddress* out_addr, size_t num_pages, size_t alignment, + KPhysicalAddress phys_addr, bool is_pa_valid, + KProcessAddress region_start, size_t region_num_pages, KMemoryState state, KMemoryPermission perm) { ASSERT(Common::IsAligned(alignment, PageSize) && alignment >= PageSize); @@ -2084,26 +2232,30 @@ Result KPageTable::MapPages(VAddr* out_addr, size_t num_pages, size_t alignment, KScopedLightLock lk(m_general_lock); // Find a random address to map at. - VAddr addr = this->FindFreeArea(region_start, region_num_pages, num_pages, alignment, 0, - this->GetNumGuardPages()); + KProcessAddress addr = this->FindFreeArea(region_start, region_num_pages, num_pages, alignment, + 0, this->GetNumGuardPages()); R_UNLESS(addr != 0, ResultOutOfMemory); - ASSERT(Common::IsAligned(addr, alignment)); + ASSERT(Common::IsAligned(GetInteger(addr), alignment)); ASSERT(this->CanContain(addr, num_pages * PageSize, state)); ASSERT(this->CheckMemoryState(addr, num_pages * PageSize, KMemoryState::All, KMemoryState::Free, KMemoryPermission::None, KMemoryPermission::None, - KMemoryAttribute::None, KMemoryAttribute::None) - .IsSuccess()); + KMemoryAttribute::None, KMemoryAttribute::None) == ResultSuccess); // Create an update allocator. - Result allocator_result{ResultSuccess}; + Result allocator_result; KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), m_memory_block_slab_manager); + R_TRY(allocator_result); + + // We're going to perform an update, so create a helper. + KScopedPageTableUpdater updater(this); // Perform mapping operation. if (is_pa_valid) { - R_TRY(this->Operate(addr, num_pages, perm, OperationType::Map, phys_addr)); + const KPageProperties properties = {perm, false, false, DisableMergeAttribute::DisableHead}; + R_TRY(this->Operate(addr, num_pages, properties.perm, OperationType::Map, phys_addr)); } else { - UNIMPLEMENTED(); + R_TRY(this->AllocateAndMapPagesImpl(updater.GetPageList(), addr, num_pages, perm)); } // Update the blocks. @@ -2116,28 +2268,45 @@ Result KPageTable::MapPages(VAddr* out_addr, size_t num_pages, size_t alignment, R_SUCCEED(); } -Result KPageTable::UnmapPages(VAddr addr, const KPageGroup& page_linked_list) { - ASSERT(this->IsLockedByCurrentThread()); +Result KPageTable::MapPages(KProcessAddress address, size_t num_pages, KMemoryState state, + KMemoryPermission perm) { + // Check that the map is in range. + const size_t size = num_pages * PageSize; + R_UNLESS(this->CanContain(address, size, state), ResultInvalidCurrentMemory); - VAddr cur_addr{addr}; + // Lock the table. + KScopedLightLock lk(m_general_lock); - for (const auto& node : page_linked_list) { - if (const auto result{Operate(cur_addr, node.GetNumPages(), KMemoryPermission::None, - OperationType::Unmap)}; - result.IsError()) { - R_RETURN(result); - } + // Check the memory state. + size_t num_allocator_blocks; + R_TRY(this->CheckMemoryState(std::addressof(num_allocator_blocks), address, size, + KMemoryState::All, KMemoryState::Free, KMemoryPermission::None, + KMemoryPermission::None, KMemoryAttribute::None, + KMemoryAttribute::None)); - cur_addr += node.GetNumPages() * PageSize; - } + // Create an update allocator. + Result allocator_result; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + m_memory_block_slab_manager, num_allocator_blocks); + R_TRY(allocator_result); + + // We're going to perform an update, so create a helper. + KScopedPageTableUpdater updater(this); + + // Map the pages. + R_TRY(this->AllocateAndMapPagesImpl(updater.GetPageList(), address, num_pages, perm)); + + // Update the blocks. + m_memory_block_manager.Update(std::addressof(allocator), address, num_pages, state, perm, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, + KMemoryBlockDisableMergeAttribute::None); R_SUCCEED(); } -Result KPageTable::UnmapPages(VAddr address, KPageGroup& page_linked_list, KMemoryState state) { +Result KPageTable::UnmapPages(KProcessAddress address, size_t num_pages, KMemoryState state) { // Check that the unmap is in range. - const size_t num_pages{page_linked_list.GetNumPages()}; - const size_t size{num_pages * PageSize}; + const size_t size = num_pages * PageSize; R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory); // Lock the table. @@ -2151,13 +2320,18 @@ Result KPageTable::UnmapPages(VAddr address, KPageGroup& page_linked_list, KMemo KMemoryAttribute::None)); // Create an update allocator. - Result allocator_result{ResultSuccess}; + Result allocator_result; KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), m_memory_block_slab_manager, num_allocator_blocks); R_TRY(allocator_result); + // We're going to perform an update, so create a helper. + KScopedPageTableUpdater updater(this); + // Perform the unmap. - R_TRY(UnmapPages(address, page_linked_list)); + const KPageProperties unmap_properties = {KMemoryPermission::None, false, false, + DisableMergeAttribute::None}; + R_TRY(this->Operate(address, num_pages, unmap_properties.perm, OperationType::Unmap)); // Update the blocks. m_memory_block_manager.Update(std::addressof(allocator), address, num_pages, KMemoryState::Free, @@ -2168,29 +2342,130 @@ Result KPageTable::UnmapPages(VAddr address, KPageGroup& page_linked_list, KMemo R_SUCCEED(); } -Result KPageTable::UnmapPages(VAddr address, size_t num_pages, KMemoryState state) { - // Check that the unmap is in range. +Result KPageTable::MapPageGroup(KProcessAddress* out_addr, const KPageGroup& pg, + KProcessAddress region_start, size_t region_num_pages, + KMemoryState state, KMemoryPermission perm) { + ASSERT(!this->IsLockedByCurrentThread()); + + // Ensure this is a valid map request. + const size_t num_pages = pg.GetNumPages(); + R_UNLESS(this->CanContain(region_start, region_num_pages * PageSize, state), + ResultInvalidCurrentMemory); + R_UNLESS(num_pages < region_num_pages, ResultOutOfMemory); + + // Lock the table. + KScopedLightLock lk(m_general_lock); + + // Find a random address to map at. + KProcessAddress addr = this->FindFreeArea(region_start, region_num_pages, num_pages, PageSize, + 0, this->GetNumGuardPages()); + R_UNLESS(addr != 0, ResultOutOfMemory); + ASSERT(this->CanContain(addr, num_pages * PageSize, state)); + ASSERT(this->CheckMemoryState(addr, num_pages * PageSize, KMemoryState::All, KMemoryState::Free, + KMemoryPermission::None, KMemoryPermission::None, + KMemoryAttribute::None, KMemoryAttribute::None) == ResultSuccess); + + // Create an update allocator. + Result allocator_result; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + m_memory_block_slab_manager); + R_TRY(allocator_result); + + // We're going to perform an update, so create a helper. + KScopedPageTableUpdater updater(this); + + // Perform mapping operation. + const KPageProperties properties = {perm, state == KMemoryState::Io, false, + DisableMergeAttribute::DisableHead}; + R_TRY(this->MapPageGroupImpl(updater.GetPageList(), addr, pg, properties, false)); + + // Update the blocks. + m_memory_block_manager.Update(std::addressof(allocator), addr, num_pages, state, perm, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, + KMemoryBlockDisableMergeAttribute::None); + + // We successfully mapped the pages. + *out_addr = addr; + R_SUCCEED(); +} + +Result KPageTable::MapPageGroup(KProcessAddress addr, const KPageGroup& pg, KMemoryState state, + KMemoryPermission perm) { + ASSERT(!this->IsLockedByCurrentThread()); + + // Ensure this is a valid map request. + const size_t num_pages = pg.GetNumPages(); const size_t size = num_pages * PageSize; - R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory); + R_UNLESS(this->CanContain(addr, size, state), ResultInvalidCurrentMemory); // Lock the table. KScopedLightLock lk(m_general_lock); - // Check the memory state. - size_t num_allocator_blocks{}; + // Check if state allows us to map. + size_t num_allocator_blocks; + R_TRY(this->CheckMemoryState(std::addressof(num_allocator_blocks), addr, size, + KMemoryState::All, KMemoryState::Free, KMemoryPermission::None, + KMemoryPermission::None, KMemoryAttribute::None, + KMemoryAttribute::None)); + + // Create an update allocator. + Result allocator_result; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + m_memory_block_slab_manager, num_allocator_blocks); + R_TRY(allocator_result); + + // We're going to perform an update, so create a helper. + KScopedPageTableUpdater updater(this); + + // Perform mapping operation. + const KPageProperties properties = {perm, state == KMemoryState::Io, false, + DisableMergeAttribute::DisableHead}; + R_TRY(this->MapPageGroupImpl(updater.GetPageList(), addr, pg, properties, false)); + + // Update the blocks. + m_memory_block_manager.Update(std::addressof(allocator), addr, num_pages, state, perm, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, + KMemoryBlockDisableMergeAttribute::None); + + // We successfully mapped the pages. + R_SUCCEED(); +} + +Result KPageTable::UnmapPageGroup(KProcessAddress address, const KPageGroup& pg, + KMemoryState state) { + ASSERT(!this->IsLockedByCurrentThread()); + + // Ensure this is a valid unmap request. + const size_t num_pages = pg.GetNumPages(); + const size_t size = num_pages * PageSize; + R_UNLESS(this->CanContain(address, size, state), ResultInvalidCurrentMemory); + + // Lock the table. + KScopedLightLock lk(m_general_lock); + + // Check if state allows us to unmap. + size_t num_allocator_blocks; R_TRY(this->CheckMemoryState(std::addressof(num_allocator_blocks), address, size, KMemoryState::All, state, KMemoryPermission::None, KMemoryPermission::None, KMemoryAttribute::All, KMemoryAttribute::None)); + // Check that the page group is valid. + R_UNLESS(this->IsValidPageGroup(pg, address, num_pages), ResultInvalidCurrentMemory); + // Create an update allocator. - Result allocator_result{ResultSuccess}; + Result allocator_result; KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), m_memory_block_slab_manager, num_allocator_blocks); R_TRY(allocator_result); - // Perform the unmap. - R_TRY(Operate(address, num_pages, KMemoryPermission::None, OperationType::Unmap)); + // We're going to perform an update, so create a helper. + KScopedPageTableUpdater updater(this); + + // Perform unmapping operation. + const KPageProperties properties = {KMemoryPermission::None, false, false, + DisableMergeAttribute::None}; + R_TRY(this->Operate(address, num_pages, properties.perm, OperationType::Unmap)); // Update the blocks. m_memory_block_manager.Update(std::addressof(allocator), address, num_pages, KMemoryState::Free, @@ -2201,7 +2476,7 @@ Result KPageTable::UnmapPages(VAddr address, size_t num_pages, KMemoryState stat R_SUCCEED(); } -Result KPageTable::MakeAndOpenPageGroup(KPageGroup* out, VAddr address, size_t num_pages, +Result KPageTable::MakeAndOpenPageGroup(KPageGroup* out, KProcessAddress address, size_t num_pages, KMemoryState state_mask, KMemoryState state, KMemoryPermission perm_mask, KMemoryPermission perm, KMemoryAttribute attr_mask, KMemoryAttribute attr) { @@ -2226,7 +2501,7 @@ Result KPageTable::MakeAndOpenPageGroup(KPageGroup* out, VAddr address, size_t n R_SUCCEED(); } -Result KPageTable::SetProcessMemoryPermission(VAddr addr, size_t size, +Result KPageTable::SetProcessMemoryPermission(KProcessAddress addr, size_t size, Svc::MemoryPermission svc_perm) { const size_t num_pages = size / PageSize; @@ -2287,23 +2562,23 @@ Result KPageTable::SetProcessMemoryPermission(VAddr addr, size_t size, // Ensure cache coherency, if we're setting pages as executable. if (is_x) { - m_system.InvalidateCpuInstructionCacheRange(addr, size); + m_system.InvalidateCpuInstructionCacheRange(GetInteger(addr), size); } R_SUCCEED(); } -KMemoryInfo KPageTable::QueryInfoImpl(VAddr addr) { +KMemoryInfo KPageTable::QueryInfoImpl(KProcessAddress addr) { KScopedLightLock lk(m_general_lock); return m_memory_block_manager.FindBlock(addr)->GetMemoryInfo(); } -KMemoryInfo KPageTable::QueryInfo(VAddr addr) { +KMemoryInfo KPageTable::QueryInfo(KProcessAddress addr) { if (!Contains(addr, 1)) { return { - .m_address = m_address_space_end, - .m_size = 0 - m_address_space_end, + .m_address = GetInteger(m_address_space_end), + .m_size = 0 - GetInteger(m_address_space_end), .m_state = static_cast<KMemoryState>(Svc::MemoryState::Inaccessible), .m_device_disable_merge_left_count = 0, .m_device_disable_merge_right_count = 0, @@ -2320,7 +2595,8 @@ KMemoryInfo KPageTable::QueryInfo(VAddr addr) { return QueryInfoImpl(addr); } -Result KPageTable::SetMemoryPermission(VAddr addr, size_t size, Svc::MemoryPermission svc_perm) { +Result KPageTable::SetMemoryPermission(KProcessAddress addr, size_t size, + Svc::MemoryPermission svc_perm) { const size_t num_pages = size / PageSize; // Lock the table. @@ -2357,7 +2633,7 @@ Result KPageTable::SetMemoryPermission(VAddr addr, size_t size, Svc::MemoryPermi R_SUCCEED(); } -Result KPageTable::SetMemoryAttribute(VAddr addr, size_t size, u32 mask, u32 attr) { +Result KPageTable::SetMemoryAttribute(KProcessAddress addr, size_t size, u32 mask, u32 attr) { const size_t num_pages = size / PageSize; ASSERT((static_cast<KMemoryAttribute>(mask) | KMemoryAttribute::SetMask) == KMemoryAttribute::SetMask); @@ -2412,12 +2688,12 @@ Result KPageTable::SetMaxHeapSize(size_t size) { R_SUCCEED(); } -Result KPageTable::SetHeapSize(VAddr* out, size_t size) { +Result KPageTable::SetHeapSize(u64* out, size_t size) { // Lock the physical memory mutex. KScopedLightLock map_phys_mem_lk(m_map_physical_memory_lock); // Try to perform a reduction in heap, instead of an extension. - VAddr cur_address{}; + KProcessAddress cur_address{}; size_t allocation_size{}; { // Lock the table. @@ -2468,11 +2744,11 @@ Result KPageTable::SetHeapSize(VAddr* out, size_t size) { m_current_heap_end = m_heap_region_start + size; // Set the output. - *out = m_heap_region_start; + *out = GetInteger(m_heap_region_start); R_SUCCEED(); } else if (size == GetHeapSize()) { // The size requested is exactly the current size. - *out = m_heap_region_start; + *out = GetInteger(m_heap_region_start); R_SUCCEED(); } else { // We have to allocate memory. Determine how much to allocate and where while the table @@ -2526,7 +2802,7 @@ Result KPageTable::SetHeapSize(VAddr* out, size_t size) { // Clear all the newly allocated pages. for (size_t cur_page = 0; cur_page < num_pages; ++cur_page) { - std::memset(m_system.Memory().GetPointer(m_current_heap_end + (cur_page * PageSize)), 0, + std::memset(m_memory->GetPointer(m_current_heap_end + (cur_page * PageSize)), 0, PageSize); } @@ -2545,62 +2821,14 @@ Result KPageTable::SetHeapSize(VAddr* out, size_t size) { m_current_heap_end = m_heap_region_start + size; // Set the output. - *out = m_heap_region_start; + *out = GetInteger(m_heap_region_start); R_SUCCEED(); } } -ResultVal<VAddr> KPageTable::AllocateAndMapMemory(size_t needed_num_pages, size_t align, - bool is_map_only, VAddr region_start, - size_t region_num_pages, KMemoryState state, - KMemoryPermission perm, PAddr map_addr) { - KScopedLightLock lk(m_general_lock); - - R_UNLESS(CanContain(region_start, region_num_pages * PageSize, state), - ResultInvalidCurrentMemory); - R_UNLESS(region_num_pages > needed_num_pages, ResultOutOfMemory); - const VAddr addr{ - AllocateVirtualMemory(region_start, region_num_pages, needed_num_pages, align)}; - R_UNLESS(addr, ResultOutOfMemory); - - // Create an update allocator. - Result allocator_result{ResultSuccess}; - KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), - m_memory_block_slab_manager); - - if (is_map_only) { - R_TRY(Operate(addr, needed_num_pages, perm, OperationType::Map, map_addr)); - } else { - // Create a page group tohold the pages we allocate. - KPageGroup pg{m_kernel, m_block_info_manager}; - - R_TRY(m_system.Kernel().MemoryManager().AllocateAndOpen( - &pg, needed_num_pages, - KMemoryManager::EncodeOption(m_memory_pool, m_allocation_option))); - - // Ensure that the page group is closed when we're done working with it. - SCOPE_EXIT({ pg.Close(); }); - - // Clear all pages. - for (const auto& it : pg) { - std::memset(m_system.DeviceMemory().GetPointer<void>(it.GetAddress()), - m_heap_fill_value, it.GetSize()); - } - - R_TRY(Operate(addr, needed_num_pages, pg, OperationType::MapGroup)); - } - - // Update the blocks. - m_memory_block_manager.Update(std::addressof(allocator), addr, needed_num_pages, state, perm, - KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, - KMemoryBlockDisableMergeAttribute::None); - - return addr; -} - -Result KPageTable::LockForMapDeviceAddressSpace(bool* out_is_io, VAddr address, size_t size, - KMemoryPermission perm, bool is_aligned, - bool check_heap) { +Result KPageTable::LockForMapDeviceAddressSpace(bool* out_is_io, KProcessAddress address, + size_t size, KMemoryPermission perm, + bool is_aligned, bool check_heap) { // Lightly validate the range before doing anything else. const size_t num_pages = size / PageSize; R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory); @@ -2636,7 +2864,8 @@ Result KPageTable::LockForMapDeviceAddressSpace(bool* out_is_io, VAddr address, R_SUCCEED(); } -Result KPageTable::LockForUnmapDeviceAddressSpace(VAddr address, size_t size, bool check_heap) { +Result KPageTable::LockForUnmapDeviceAddressSpace(KProcessAddress address, size_t size, + bool check_heap) { // Lightly validate the range before doing anything else. const size_t num_pages = size / PageSize; R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory); @@ -2670,7 +2899,7 @@ Result KPageTable::LockForUnmapDeviceAddressSpace(VAddr address, size_t size, bo R_SUCCEED(); } -Result KPageTable::UnlockForDeviceAddressSpace(VAddr address, size_t size) { +Result KPageTable::UnlockForDeviceAddressSpace(KProcessAddress address, size_t size) { // Lightly validate the range before doing anything else. const size_t num_pages = size / PageSize; R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory); @@ -2698,7 +2927,8 @@ Result KPageTable::UnlockForDeviceAddressSpace(VAddr address, size_t size) { R_SUCCEED(); } -Result KPageTable::LockForIpcUserBuffer(PAddr* out, VAddr address, size_t size) { +Result KPageTable::LockForIpcUserBuffer(KPhysicalAddress* out, KProcessAddress address, + size_t size) { R_RETURN(this->LockMemoryAndOpen( nullptr, out, address, size, KMemoryState::FlagCanIpcUserBuffer, KMemoryState::FlagCanIpcUserBuffer, KMemoryPermission::All, @@ -2707,7 +2937,7 @@ Result KPageTable::LockForIpcUserBuffer(PAddr* out, VAddr address, size_t size) KMemoryAttribute::Locked)); } -Result KPageTable::UnlockForIpcUserBuffer(VAddr address, size_t size) { +Result KPageTable::UnlockForIpcUserBuffer(KProcessAddress address, size_t size) { R_RETURN(this->UnlockMemory(address, size, KMemoryState::FlagCanIpcUserBuffer, KMemoryState::FlagCanIpcUserBuffer, KMemoryPermission::None, KMemoryPermission::None, KMemoryAttribute::All, @@ -2715,7 +2945,7 @@ Result KPageTable::UnlockForIpcUserBuffer(VAddr address, size_t size) { KMemoryAttribute::Locked, nullptr)); } -Result KPageTable::LockForCodeMemory(KPageGroup* out, VAddr addr, size_t size) { +Result KPageTable::LockForCodeMemory(KPageGroup* out, KProcessAddress addr, size_t size) { R_RETURN(this->LockMemoryAndOpen( out, nullptr, addr, size, KMemoryState::FlagCanCodeMemory, KMemoryState::FlagCanCodeMemory, KMemoryPermission::All, KMemoryPermission::UserReadWrite, KMemoryAttribute::All, @@ -2723,17 +2953,17 @@ Result KPageTable::LockForCodeMemory(KPageGroup* out, VAddr addr, size_t size) { KMemoryAttribute::Locked)); } -Result KPageTable::UnlockForCodeMemory(VAddr addr, size_t size, const KPageGroup& pg) { +Result KPageTable::UnlockForCodeMemory(KProcessAddress addr, size_t size, const KPageGroup& pg) { R_RETURN(this->UnlockMemory( addr, size, KMemoryState::FlagCanCodeMemory, KMemoryState::FlagCanCodeMemory, KMemoryPermission::None, KMemoryPermission::None, KMemoryAttribute::All, KMemoryAttribute::Locked, KMemoryPermission::UserReadWrite, KMemoryAttribute::Locked, &pg)); } -bool KPageTable::IsRegionContiguous(VAddr addr, u64 size) const { - auto start_ptr = m_system.DeviceMemory().GetPointer<u8>(addr); +bool KPageTable::IsRegionContiguous(KProcessAddress addr, u64 size) const { + auto start_ptr = m_system.DeviceMemory().GetPointer<u8>(GetInteger(addr)); for (u64 offset{}; offset < size; offset += PageSize) { - if (start_ptr != m_system.DeviceMemory().GetPointer<u8>(addr + offset)) { + if (start_ptr != m_system.DeviceMemory().GetPointer<u8>(GetInteger(addr) + offset)) { return false; } start_ptr += PageSize; @@ -2741,18 +2971,19 @@ bool KPageTable::IsRegionContiguous(VAddr addr, u64 size) const { return true; } -void KPageTable::AddRegionToPages(VAddr start, size_t num_pages, KPageGroup& page_linked_list) { - VAddr addr{start}; +void KPageTable::AddRegionToPages(KProcessAddress start, size_t num_pages, + KPageGroup& page_linked_list) { + KProcessAddress addr{start}; while (addr < start + (num_pages * PageSize)) { - const PAddr paddr{GetPhysicalAddr(addr)}; + const KPhysicalAddress paddr{GetPhysicalAddr(addr)}; ASSERT(paddr != 0); page_linked_list.AddBlock(paddr, 1); addr += PageSize; } } -VAddr KPageTable::AllocateVirtualMemory(VAddr start, size_t region_num_pages, u64 needed_num_pages, - size_t align) { +KProcessAddress KPageTable::AllocateVirtualMemory(KProcessAddress start, size_t region_num_pages, + u64 needed_num_pages, size_t align) { if (m_enable_aslr) { UNIMPLEMENTED(); } @@ -2760,11 +2991,11 @@ VAddr KPageTable::AllocateVirtualMemory(VAddr start, size_t region_num_pages, u6 IsKernel() ? 1 : 4); } -Result KPageTable::Operate(VAddr addr, size_t num_pages, const KPageGroup& page_group, +Result KPageTable::Operate(KProcessAddress addr, size_t num_pages, const KPageGroup& page_group, OperationType operation) { ASSERT(this->IsLockedByCurrentThread()); - ASSERT(Common::IsAligned(addr, PageSize)); + ASSERT(Common::IsAligned(GetInteger(addr), PageSize)); ASSERT(num_pages > 0); ASSERT(num_pages == page_group.GetNumPages()); @@ -2777,7 +3008,7 @@ Result KPageTable::Operate(VAddr addr, size_t num_pages, const KPageGroup& page_ const size_t size{node.GetNumPages() * PageSize}; // Map the pages. - m_system.Memory().MapMemoryRegion(*m_page_table_impl, addr, size, node.GetAddress()); + m_memory->MapMemoryRegion(*m_page_table_impl, addr, size, node.GetAddress()); addr += size; } @@ -2795,12 +3026,12 @@ Result KPageTable::Operate(VAddr addr, size_t num_pages, const KPageGroup& page_ R_SUCCEED(); } -Result KPageTable::Operate(VAddr addr, size_t num_pages, KMemoryPermission perm, - OperationType operation, PAddr map_addr) { +Result KPageTable::Operate(KProcessAddress addr, size_t num_pages, KMemoryPermission perm, + OperationType operation, KPhysicalAddress map_addr) { ASSERT(this->IsLockedByCurrentThread()); ASSERT(num_pages > 0); - ASSERT(Common::IsAligned(addr, PageSize)); + ASSERT(Common::IsAligned(GetInteger(addr), PageSize)); ASSERT(ContainsPages(addr, num_pages)); switch (operation) { @@ -2810,14 +3041,14 @@ Result KPageTable::Operate(VAddr addr, size_t num_pages, KMemoryPermission perm, SCOPE_EXIT({ pages_to_close.CloseAndReset(); }); this->AddRegionToPages(addr, num_pages, pages_to_close); - m_system.Memory().UnmapRegion(*m_page_table_impl, addr, num_pages * PageSize); + m_memory->UnmapRegion(*m_page_table_impl, addr, num_pages * PageSize); break; } case OperationType::MapFirst: case OperationType::Map: { ASSERT(map_addr); - ASSERT(Common::IsAligned(map_addr, PageSize)); - m_system.Memory().MapMemoryRegion(*m_page_table_impl, addr, num_pages * PageSize, map_addr); + ASSERT(Common::IsAligned(GetInteger(map_addr), PageSize)); + m_memory->MapMemoryRegion(*m_page_table_impl, addr, num_pages * PageSize, map_addr); // Open references to pages, if we should. if (IsHeapPhysicalAddress(m_kernel.MemoryLayout(), map_addr)) { @@ -2854,7 +3085,7 @@ void KPageTable::FinalizeUpdate(PageLinkedList* page_list) { } } -VAddr KPageTable::GetRegionAddress(KMemoryState state) const { +KProcessAddress KPageTable::GetRegionAddress(KMemoryState state) const { switch (state) { case KMemoryState::Free: case KMemoryState::Kernel: @@ -2926,11 +3157,11 @@ size_t KPageTable::GetRegionSize(KMemoryState state) const { } } -bool KPageTable::CanContain(VAddr addr, size_t size, KMemoryState state) const { - const VAddr end = addr + size; - const VAddr last = end - 1; +bool KPageTable::CanContain(KProcessAddress addr, size_t size, KMemoryState state) const { + const KProcessAddress end = addr + size; + const KProcessAddress last = end - 1; - const VAddr region_start = this->GetRegionAddress(state); + const KProcessAddress region_start = this->GetRegionAddress(state); const size_t region_size = this->GetRegionSize(state); const bool is_in_region = @@ -2985,21 +3216,21 @@ Result KPageTable::CheckMemoryState(const KMemoryInfo& info, KMemoryState state_ R_SUCCEED(); } -Result KPageTable::CheckMemoryStateContiguous(size_t* out_blocks_needed, VAddr addr, size_t size, - KMemoryState state_mask, KMemoryState state, - KMemoryPermission perm_mask, KMemoryPermission perm, - KMemoryAttribute attr_mask, +Result KPageTable::CheckMemoryStateContiguous(size_t* out_blocks_needed, KProcessAddress addr, + size_t size, KMemoryState state_mask, + KMemoryState state, KMemoryPermission perm_mask, + KMemoryPermission perm, KMemoryAttribute attr_mask, KMemoryAttribute attr) const { ASSERT(this->IsLockedByCurrentThread()); // Get information about the first block. - const VAddr last_addr = addr + size - 1; + const KProcessAddress last_addr = addr + size - 1; KMemoryBlockManager::const_iterator it = m_memory_block_manager.FindIterator(addr); KMemoryInfo info = it->GetMemoryInfo(); // If the start address isn't aligned, we need a block. const size_t blocks_for_start_align = - (Common::AlignDown(addr, PageSize) != info.GetAddress()) ? 1 : 0; + (Common::AlignDown(GetInteger(addr), PageSize) != info.GetAddress()) ? 1 : 0; while (true) { // Validate against the provided masks. @@ -3018,7 +3249,7 @@ Result KPageTable::CheckMemoryStateContiguous(size_t* out_blocks_needed, VAddr a // If the end address isn't aligned, we need a block. const size_t blocks_for_end_align = - (Common::AlignUp(addr + size, PageSize) != info.GetEndAddress()) ? 1 : 0; + (Common::AlignUp(GetInteger(addr) + size, PageSize) != info.GetEndAddress()) ? 1 : 0; if (out_blocks_needed != nullptr) { *out_blocks_needed = blocks_for_start_align + blocks_for_end_align; @@ -3029,20 +3260,20 @@ Result KPageTable::CheckMemoryStateContiguous(size_t* out_blocks_needed, VAddr a Result KPageTable::CheckMemoryState(KMemoryState* out_state, KMemoryPermission* out_perm, KMemoryAttribute* out_attr, size_t* out_blocks_needed, - VAddr addr, size_t size, KMemoryState state_mask, + KProcessAddress addr, size_t size, KMemoryState state_mask, KMemoryState state, KMemoryPermission perm_mask, KMemoryPermission perm, KMemoryAttribute attr_mask, KMemoryAttribute attr, KMemoryAttribute ignore_attr) const { ASSERT(this->IsLockedByCurrentThread()); // Get information about the first block. - const VAddr last_addr = addr + size - 1; + const KProcessAddress last_addr = addr + size - 1; KMemoryBlockManager::const_iterator it = m_memory_block_manager.FindIterator(addr); KMemoryInfo info = it->GetMemoryInfo(); // If the start address isn't aligned, we need a block. const size_t blocks_for_start_align = - (Common::AlignDown(addr, PageSize) != info.GetAddress()) ? 1 : 0; + (Common::AlignDown(GetInteger(addr), PageSize) != info.GetAddress()) ? 1 : 0; // Validate all blocks in the range have correct state. const KMemoryState first_state = info.m_state; @@ -3071,7 +3302,7 @@ Result KPageTable::CheckMemoryState(KMemoryState* out_state, KMemoryPermission* // If the end address isn't aligned, we need a block. const size_t blocks_for_end_align = - (Common::AlignUp(addr + size, PageSize) != info.GetEndAddress()) ? 1 : 0; + (Common::AlignUp(GetInteger(addr) + size, PageSize) != info.GetEndAddress()) ? 1 : 0; // Write output state. if (out_state != nullptr) { @@ -3089,11 +3320,12 @@ Result KPageTable::CheckMemoryState(KMemoryState* out_state, KMemoryPermission* R_SUCCEED(); } -Result KPageTable::LockMemoryAndOpen(KPageGroup* out_pg, PAddr* out_paddr, VAddr addr, size_t size, - KMemoryState state_mask, KMemoryState state, - KMemoryPermission perm_mask, KMemoryPermission perm, - KMemoryAttribute attr_mask, KMemoryAttribute attr, - KMemoryPermission new_perm, KMemoryAttribute lock_attr) { +Result KPageTable::LockMemoryAndOpen(KPageGroup* out_pg, KPhysicalAddress* out_KPhysicalAddress, + KProcessAddress addr, size_t size, KMemoryState state_mask, + KMemoryState state, KMemoryPermission perm_mask, + KMemoryPermission perm, KMemoryAttribute attr_mask, + KMemoryAttribute attr, KMemoryPermission new_perm, + KMemoryAttribute lock_attr) { // Validate basic preconditions. ASSERT((lock_attr & attr) == KMemoryAttribute::None); ASSERT((lock_attr & (KMemoryAttribute::IpcLocked | KMemoryAttribute::DeviceShared)) == @@ -3123,8 +3355,8 @@ Result KPageTable::LockMemoryAndOpen(KPageGroup* out_pg, PAddr* out_paddr, VAddr attr_mask, attr)); // Get the physical address, if we're supposed to. - if (out_paddr != nullptr) { - ASSERT(this->GetPhysicalAddressLocked(out_paddr, addr)); + if (out_KPhysicalAddress != nullptr) { + ASSERT(this->GetPhysicalAddressLocked(out_KPhysicalAddress, addr)); } // Make the page group, if we're supposed to. @@ -3155,7 +3387,7 @@ Result KPageTable::LockMemoryAndOpen(KPageGroup* out_pg, PAddr* out_paddr, VAddr R_SUCCEED(); } -Result KPageTable::UnlockMemory(VAddr addr, size_t size, KMemoryState state_mask, +Result KPageTable::UnlockMemory(KProcessAddress addr, size_t size, KMemoryState state_mask, KMemoryState state, KMemoryPermission perm_mask, KMemoryPermission perm, KMemoryAttribute attr_mask, KMemoryAttribute attr, KMemoryPermission new_perm, |