From 738f91a57da7c129d1ee85b7abbf6858f8669ee3 Mon Sep 17 00:00:00 2001 From: MerryMage Date: Sat, 27 Jan 2018 15:16:39 +0000 Subject: memory: Replace all memory hooking with Special regions --- src/core/CMakeLists.txt | 2 +- src/core/hle/kernel/vm_manager.cpp | 6 +- src/core/hle/kernel/vm_manager.h | 6 +- src/core/memory.cpp | 480 +++++++-------------- src/core/memory.h | 72 +--- src/core/memory_hook.h | 46 ++ src/core/memory_setup.h | 8 +- src/core/mmio.h | 38 -- src/tests/core/arm/arm_test_common.cpp | 32 +- src/tests/core/arm/arm_test_common.h | 22 +- src/video_core/renderer_opengl/renderer_opengl.cpp | 2 +- 11 files changed, 273 insertions(+), 441 deletions(-) create mode 100644 src/core/memory_hook.h delete mode 100644 src/core/mmio.h (limited to 'src') diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 7153c4f3f..14bc99ba6 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -163,8 +163,8 @@ add_library(core STATIC loader/nso.h memory.cpp memory.h + memory_hook.h memory_setup.h - mmio.h perf_stats.cpp perf_stats.h settings.cpp diff --git a/src/core/hle/kernel/vm_manager.cpp b/src/core/hle/kernel/vm_manager.cpp index 93662a45e..6da77eb58 100644 --- a/src/core/hle/kernel/vm_manager.cpp +++ b/src/core/hle/kernel/vm_manager.cpp @@ -10,8 +10,8 @@ #include "core/hle/kernel/errors.h" #include "core/hle/kernel/vm_manager.h" #include "core/memory.h" +#include "core/memory_hook.h" #include "core/memory_setup.h" -#include "core/mmio.h" namespace Kernel { @@ -60,8 +60,8 @@ void VMManager::Reset() { vma_map.emplace(initial_vma.base, initial_vma); page_table.pointers.fill(nullptr); + page_table.special_regions.clear(); page_table.attributes.fill(Memory::PageType::Unmapped); - page_table.cached_res_count.fill(0); UpdatePageTableForVMA(initial_vma); } @@ -121,7 +121,7 @@ ResultVal VMManager::MapBackingMemory(VAddr target, u8* me ResultVal VMManager::MapMMIO(VAddr target, PAddr paddr, u64 size, MemoryState state, - Memory::MMIORegionPointer mmio_handler) { + Memory::MemoryHookPointer mmio_handler) { // This is the appropriately sized VMA that will turn into our allocation. CASCADE_RESULT(VMAIter vma_handle, CarveVMA(target, size)); VirtualMemoryArea& final_vma = vma_handle->second; diff --git a/src/core/hle/kernel/vm_manager.h b/src/core/hle/kernel/vm_manager.h index b17385c7c..8de704a60 100644 --- a/src/core/hle/kernel/vm_manager.h +++ b/src/core/hle/kernel/vm_manager.h @@ -10,7 +10,7 @@ #include "common/common_types.h" #include "core/hle/result.h" #include "core/memory.h" -#include "core/mmio.h" +#include "core/memory_hook.h" namespace Kernel { @@ -81,7 +81,7 @@ struct VirtualMemoryArea { // Settings for type = MMIO /// Physical address of the register area this VMA maps to. PAddr paddr = 0; - Memory::MMIORegionPointer mmio_handler = nullptr; + Memory::MemoryHookPointer mmio_handler = nullptr; /// Tests if this area can be merged to the right with `next`. bool CanBeMergedWith(const VirtualMemoryArea& next) const; @@ -160,7 +160,7 @@ public: * @param mmio_handler The handler that will implement read and write for this MMIO region. */ ResultVal MapMMIO(VAddr target, PAddr paddr, u64 size, MemoryState state, - Memory::MMIORegionPointer mmio_handler); + Memory::MemoryHookPointer mmio_handler); /// Unmaps a range of addresses, splitting VMAs as necessary. ResultCode UnmapRange(VAddr target, u64 size); diff --git a/src/core/memory.cpp b/src/core/memory.cpp index a3d2d4951..f658271a5 100644 --- a/src/core/memory.cpp +++ b/src/core/memory.cpp @@ -2,8 +2,10 @@ // Licensed under GPLv2 or any later version // Refer to the license.txt file included. +#include #include #include +#include #include "common/assert.h" #include "common/common_types.h" #include "common/logging/log.h" @@ -12,7 +14,6 @@ #include "core/core.h" #include "core/hle/kernel/memory.h" #include "core/hle/kernel/process.h" -#include "core/hle/lock.h" #include "core/memory.h" #include "core/memory_setup.h" #include "video_core/renderer_base.h" @@ -40,16 +41,12 @@ static void MapPages(PageTable& page_table, VAddr base, u64 size, u8* memory, Pa LOG_DEBUG(HW_Memory, "Mapping %p onto %08X-%08X", memory, base * PAGE_SIZE, (base + size) * PAGE_SIZE); - RasterizerFlushVirtualRegion(base << PAGE_BITS, size * PAGE_SIZE, - FlushMode::FlushAndInvalidate); - VAddr end = base + size; while (base != end) { ASSERT_MSG(base < PAGE_TABLE_NUM_ENTRIES, "out of range mapping at %08X", base); page_table.attributes[base] = type; page_table.pointers[base] = memory; - page_table.cached_res_count[base] = 0; base += 1; if (memory != nullptr) @@ -63,157 +60,110 @@ void MapMemoryRegion(PageTable& page_table, VAddr base, u64 size, u8* target) { MapPages(page_table, base / PAGE_SIZE, size / PAGE_SIZE, target, PageType::Memory); } -void MapIoRegion(PageTable& page_table, VAddr base, u64 size, MMIORegionPointer mmio_handler) { +void MapIoRegion(PageTable& page_table, VAddr base, u64 size, MemoryHookPointer mmio_handler) { ASSERT_MSG((size & PAGE_MASK) == 0, "non-page aligned size: %08X", size); ASSERT_MSG((base & PAGE_MASK) == 0, "non-page aligned base: %08X", base); MapPages(page_table, base / PAGE_SIZE, size / PAGE_SIZE, nullptr, PageType::Special); - page_table.special_regions.emplace_back(SpecialRegion{base, size, mmio_handler}); + auto interval = boost::icl::discrete_interval::closed(base, base + size - 1); + SpecialRegion region{SpecialRegion::Type::IODevice, mmio_handler}; + page_table.special_regions.add(std::make_pair(interval, std::set{region})); } void UnmapRegion(PageTable& page_table, VAddr base, u64 size) { ASSERT_MSG((size & PAGE_MASK) == 0, "non-page aligned size: %08X", size); ASSERT_MSG((base & PAGE_MASK) == 0, "non-page aligned base: %08X", base); MapPages(page_table, base / PAGE_SIZE, size / PAGE_SIZE, nullptr, PageType::Unmapped); -} - -/** - * Gets a pointer to the exact memory at the virtual address (i.e. not page aligned) - * using a VMA from the current process - */ -static u8* GetPointerFromVMA(const Kernel::Process& process, VAddr vaddr) { - u8* direct_pointer = nullptr; - auto& vm_manager = process.vm_manager; - - auto it = vm_manager.FindVMA(vaddr); - ASSERT(it != vm_manager.vma_map.end()); - - auto& vma = it->second; - switch (vma.type) { - case Kernel::VMAType::AllocatedMemoryBlock: - direct_pointer = vma.backing_block->data() + vma.offset; - break; - case Kernel::VMAType::BackingMemory: - direct_pointer = vma.backing_memory; - break; - case Kernel::VMAType::Free: - return nullptr; - default: - UNREACHABLE(); - } + auto interval = boost::icl::discrete_interval::closed(base, base + size - 1); + page_table.special_regions.erase(interval); +} - return direct_pointer + (vaddr - vma.base); +void AddDebugHook(PageTable& page_table, VAddr base, u64 size, MemoryHookPointer hook) { + auto interval = boost::icl::discrete_interval::closed(base, base + size - 1); + SpecialRegion region{SpecialRegion::Type::DebugHook, hook}; + page_table.special_regions.add(std::make_pair(interval, std::set{region})); } -/** - * Gets a pointer to the exact memory at the virtual address (i.e. not page aligned) - * using a VMA from the current process. - */ -static u8* GetPointerFromVMA(VAddr vaddr) { - return GetPointerFromVMA(*Kernel::g_current_process, vaddr); +void RemoveDebugHook(PageTable& page_table, VAddr base, u64 size, MemoryHookPointer hook) { + auto interval = boost::icl::discrete_interval::closed(base, base + size - 1); + SpecialRegion region{SpecialRegion::Type::DebugHook, hook}; + page_table.special_regions.subtract(std::make_pair(interval, std::set{region})); } /** * This function should only be called for virtual addreses with attribute `PageType::Special`. */ -static MMIORegionPointer GetMMIOHandler(const PageTable& page_table, VAddr vaddr) { - for (const auto& region : page_table.special_regions) { - if (vaddr >= region.base && vaddr < (region.base + region.size)) { - return region.handler; +static std::set GetSpecialHandlers(const PageTable& page_table, VAddr vaddr, + u64 size) { + std::set result; + auto interval = boost::icl::discrete_interval::closed(vaddr, vaddr + size - 1); + auto interval_list = page_table.special_regions.equal_range(interval); + for (auto it = interval_list.first; it != interval_list.second; ++it) { + for (const auto& region : it->second) { + result.insert(region.handler); } } - ASSERT_MSG(false, "Mapped IO page without a handler @ %08X", vaddr); - return nullptr; // Should never happen + return result; } -static MMIORegionPointer GetMMIOHandler(VAddr vaddr) { +static std::set GetSpecialHandlers(VAddr vaddr, u64 size) { const PageTable& page_table = Kernel::g_current_process->vm_manager.page_table; - return GetMMIOHandler(page_table, vaddr); + return GetSpecialHandlers(page_table, vaddr, size); } template -T ReadMMIO(MMIORegionPointer mmio_handler, VAddr addr); +boost::optional ReadSpecial(VAddr addr); template T Read(const VAddr vaddr) { - const u8* page_pointer = current_page_table->pointers[vaddr >> PAGE_BITS]; - if (page_pointer) { - // NOTE: Avoid adding any extra logic to this fast-path block - T value; - std::memcpy(&value, &page_pointer[vaddr & PAGE_MASK], sizeof(T)); - return value; - } - - // The memory access might do an MMIO or cached access, so we have to lock the HLE kernel state - std::lock_guard lock(HLE::g_hle_lock); - - PageType type = current_page_table->attributes[vaddr >> PAGE_BITS]; + const PageType type = current_page_table->attributes[vaddr >> PAGE_BITS]; switch (type) { case PageType::Unmapped: - LOG_ERROR(HW_Memory, "unmapped Read%lu @ 0x%llx", sizeof(T) * 8, vaddr); + LOG_ERROR(HW_Memory, "unmapped Read%lu @ 0x%016llX", sizeof(T) * 8, vaddr); return 0; - case PageType::Memory: - ASSERT_MSG(false, "Mapped memory page without a pointer @ %08X", vaddr); - break; - case PageType::RasterizerCachedMemory: { - RasterizerFlushVirtualRegion(vaddr, sizeof(T), FlushMode::Flush); + case PageType::Special: { + if (auto result = ReadSpecial(vaddr)) + return *result; + [[fallthrough]]; + } + case PageType::Memory: { + const u8* page_pointer = current_page_table->pointers[vaddr >> PAGE_BITS]; + ASSERT_MSG(page_pointer, "Mapped memory page without a pointer @ %08X", vaddr); T value; - std::memcpy(&value, GetPointerFromVMA(vaddr), sizeof(T)); + std::memcpy(&value, &page_pointer[vaddr & PAGE_MASK], sizeof(T)); return value; } - case PageType::Special: - return ReadMMIO(GetMMIOHandler(vaddr), vaddr); - case PageType::RasterizerCachedSpecial: { - RasterizerFlushVirtualRegion(vaddr, sizeof(T), FlushMode::Flush); - return ReadMMIO(GetMMIOHandler(vaddr), vaddr); - } - default: - UNREACHABLE(); } + UNREACHABLE(); + return 0; } template -void WriteMMIO(MMIORegionPointer mmio_handler, VAddr addr, const T data); +bool WriteSpecial(VAddr addr, const T data); template void Write(const VAddr vaddr, const T data) { - u8* page_pointer = current_page_table->pointers[vaddr >> PAGE_BITS]; - if (page_pointer) { - // NOTE: Avoid adding any extra logic to this fast-path block - std::memcpy(&page_pointer[vaddr & PAGE_MASK], &data, sizeof(T)); - return; - } - - // The memory access might do an MMIO or cached access, so we have to lock the HLE kernel state - std::lock_guard lock(HLE::g_hle_lock); - - PageType type = current_page_table->attributes[vaddr >> PAGE_BITS]; + const PageType type = current_page_table->attributes[vaddr >> PAGE_BITS]; switch (type) { case PageType::Unmapped: LOG_ERROR(HW_Memory, "unmapped Write%lu 0x%08X @ 0x%08X", sizeof(data) * 8, (u32)data, vaddr); return; - case PageType::Memory: - ASSERT_MSG(false, "Mapped memory page without a pointer @ %08X", vaddr); - break; - case PageType::RasterizerCachedMemory: { - RasterizerFlushVirtualRegion(vaddr, sizeof(T), FlushMode::FlushAndInvalidate); - std::memcpy(GetPointerFromVMA(vaddr), &data, sizeof(T)); - break; + case PageType::Special: { + if (WriteSpecial(vaddr, data)) + return; + [[fallthrough]]; } - case PageType::Special: - WriteMMIO(GetMMIOHandler(vaddr), vaddr, data); - break; - case PageType::RasterizerCachedSpecial: { - RasterizerFlushVirtualRegion(vaddr, sizeof(T), FlushMode::FlushAndInvalidate); - WriteMMIO(GetMMIOHandler(vaddr), vaddr, data); - break; + case PageType::Memory: { + u8* page_pointer = current_page_table->pointers[vaddr >> PAGE_BITS]; + ASSERT_MSG(page_pointer, "Mapped memory page without a pointer @ %08X", vaddr); + std::memcpy(&page_pointer[vaddr & PAGE_MASK], &data, sizeof(T)); + return; } - default: - UNREACHABLE(); } + UNREACHABLE(); } bool IsValidVirtualAddress(const Kernel::Process& process, const VAddr vaddr) { @@ -222,21 +172,20 @@ bool IsValidVirtualAddress(const Kernel::Process& process, const VAddr vaddr) { if ((vaddr >> PAGE_BITS) >= PAGE_TABLE_NUM_ENTRIES) return false; - const u8* page_pointer = page_table.pointers[vaddr >> PAGE_BITS]; - if (page_pointer) - return true; - - if (page_table.attributes[vaddr >> PAGE_BITS] == PageType::RasterizerCachedMemory) - return true; - - if (page_table.attributes[vaddr >> PAGE_BITS] != PageType::Special) + const PageType type = current_page_table->attributes[vaddr >> PAGE_BITS]; + switch (type) { + case PageType::Unmapped: return false; - - MMIORegionPointer mmio_region = GetMMIOHandler(page_table, vaddr); - if (mmio_region) { - return mmio_region->IsValidAddress(vaddr); + case PageType::Memory: + return true; + case PageType::Special: { + for (auto handler : GetSpecialHandlers(page_table, vaddr, 1)) + if (auto result = handler->IsValidAddress(vaddr)) + return *result; + return current_page_table->pointers[vaddr >> PAGE_BITS] != nullptr; } - + } + UNREACHABLE(); return false; } @@ -254,10 +203,6 @@ u8* GetPointer(const VAddr vaddr) { return page_pointer + (vaddr & PAGE_MASK); } - if (current_page_table->attributes[vaddr >> PAGE_BITS] == PageType::RasterizerCachedMemory) { - return GetPointerFromVMA(vaddr); - } - LOG_ERROR(HW_Memory, "unknown GetPointer @ 0x%08x", vaddr); return nullptr; } @@ -335,97 +280,6 @@ u8* GetPhysicalPointer(PAddr address) { return target_pointer; } -void RasterizerMarkRegionCached(PAddr start, u64 size, int count_delta) { - if (start == 0) { - return; - } - - u64 num_pages = ((start + size - 1) >> PAGE_BITS) - (start >> PAGE_BITS) + 1; - PAddr paddr = start; - - for (unsigned i = 0; i < num_pages; ++i, paddr += PAGE_SIZE) { - boost::optional maybe_vaddr = PhysicalToVirtualAddress(paddr); - // While the physical <-> virtual mapping is 1:1 for the regions supported by the cache, - // some games (like Pokemon Super Mystery Dungeon) will try to use textures that go beyond - // the end address of VRAM, causing the Virtual->Physical translation to fail when flushing - // parts of the texture. - if (!maybe_vaddr) { - LOG_ERROR(HW_Memory, - "Trying to flush a cached region to an invalid physical address %08X", paddr); - continue; - } - VAddr vaddr = *maybe_vaddr; - - u8& res_count = current_page_table->cached_res_count[vaddr >> PAGE_BITS]; - ASSERT_MSG(count_delta <= UINT8_MAX - res_count, - "Rasterizer resource cache counter overflow!"); - ASSERT_MSG(count_delta >= -res_count, "Rasterizer resource cache counter underflow!"); - - // Switch page type to cached if now cached - if (res_count == 0) { - PageType& page_type = current_page_table->attributes[vaddr >> PAGE_BITS]; - switch (page_type) { - case PageType::Unmapped: - // It is not necessary for a process to have this region mapped into its address - // space, for example, a system module need not have a VRAM mapping. - break; - case PageType::Memory: - page_type = PageType::RasterizerCachedMemory; - current_page_table->pointers[vaddr >> PAGE_BITS] = nullptr; - break; - case PageType::Special: - page_type = PageType::RasterizerCachedSpecial; - break; - default: - UNREACHABLE(); - } - } - - res_count += count_delta; - - // Switch page type to uncached if now uncached - if (res_count == 0) { - PageType& page_type = current_page_table->attributes[vaddr >> PAGE_BITS]; - switch (page_type) { - case PageType::Unmapped: - // It is not necessary for a process to have this region mapped into its address - // space, for example, a system module need not have a VRAM mapping. - break; - case PageType::RasterizerCachedMemory: { - u8* pointer = GetPointerFromVMA(vaddr & ~PAGE_MASK); - if (pointer == nullptr) { - // It's possible that this function has called been while updating the pagetable - // after unmapping a VMA. In that case the underlying VMA will no longer exist, - // and we should just leave the pagetable entry blank. - page_type = PageType::Unmapped; - } else { - page_type = PageType::Memory; - current_page_table->pointers[vaddr >> PAGE_BITS] = pointer; - } - break; - } - case PageType::RasterizerCachedSpecial: - page_type = PageType::Special; - break; - default: - UNREACHABLE(); - } - } - } -} - -void RasterizerFlushRegion(PAddr start, u64 size) {} - -void RasterizerFlushAndInvalidateRegion(PAddr start, u64 size) { - // Since pages are unmapped on shutdown after video core is shutdown, the renderer may be - // null here -} - -void RasterizerFlushVirtualRegion(VAddr start, u64 size, FlushMode mode) { - // Since pages are unmapped on shutdown after video core is shutdown, the renderer may be - // null here -} - u8 Read8(const VAddr addr) { return Read(addr); } @@ -442,6 +296,17 @@ u64 Read64(const VAddr addr) { return Read(addr); } +static bool ReadSpecialBlock(const Kernel::Process& process, const VAddr src_addr, + void* dest_buffer, const size_t size) { + auto& page_table = process.vm_manager.page_table; + for (const auto& handler : GetSpecialHandlers(page_table, src_addr, size)) { + if (handler->ReadBlock(src_addr, dest_buffer, size)) { + return true; + } + } + return false; +} + void ReadBlock(const Kernel::Process& process, const VAddr src_addr, void* dest_buffer, const size_t size) { auto& page_table = process.vm_manager.page_table; @@ -455,11 +320,15 @@ void ReadBlock(const Kernel::Process& process, const VAddr src_addr, void* dest_ const VAddr current_vaddr = static_cast((page_index << PAGE_BITS) + page_offset); switch (page_table.attributes[page_index]) { - case PageType::Unmapped: { + case PageType::Unmapped: LOG_ERROR(HW_Memory, "unmapped ReadBlock @ 0x%08X (start address = 0xllx, size = %zu)", current_vaddr, src_addr, size); std::memset(dest_buffer, 0, copy_amount); break; + case PageType::Special: { + if (ReadSpecialBlock(process, current_vaddr, dest_buffer, copy_amount)) + break; + [[fallthrough]]; } case PageType::Memory: { DEBUG_ASSERT(page_table.pointers[page_index]); @@ -468,26 +337,6 @@ void ReadBlock(const Kernel::Process& process, const VAddr src_addr, void* dest_ std::memcpy(dest_buffer, src_ptr, copy_amount); break; } - case PageType::Special: { - MMIORegionPointer handler = GetMMIOHandler(page_table, current_vaddr); - DEBUG_ASSERT(handler); - handler->ReadBlock(current_vaddr, dest_buffer, copy_amount); - break; - } - case PageType::RasterizerCachedMemory: { - RasterizerFlushVirtualRegion(current_vaddr, static_cast(copy_amount), - FlushMode::Flush); - std::memcpy(dest_buffer, GetPointerFromVMA(process, current_vaddr), copy_amount); - break; - } - case PageType::RasterizerCachedSpecial: { - MMIORegionPointer handler = GetMMIOHandler(page_table, current_vaddr); - DEBUG_ASSERT(handler); - RasterizerFlushVirtualRegion(current_vaddr, static_cast(copy_amount), - FlushMode::Flush); - handler->ReadBlock(current_vaddr, dest_buffer, copy_amount); - break; - } default: UNREACHABLE(); } @@ -519,6 +368,17 @@ void Write64(const VAddr addr, const u64 data) { Write(addr, data); } +static bool WriteSpecialBlock(const Kernel::Process& process, const VAddr dest_addr, + const void* src_buffer, const size_t size) { + auto& page_table = process.vm_manager.page_table; + for (const auto& handler : GetSpecialHandlers(page_table, dest_addr, size)) { + if (handler->WriteBlock(dest_addr, src_buffer, size)) { + return true; + } + } + return false; +} + void WriteBlock(const Kernel::Process& process, const VAddr dest_addr, const void* src_buffer, const size_t size) { auto& page_table = process.vm_manager.page_table; @@ -531,12 +391,15 @@ void WriteBlock(const Kernel::Process& process, const VAddr dest_addr, const voi const VAddr current_vaddr = static_cast((page_index << PAGE_BITS) + page_offset); switch (page_table.attributes[page_index]) { - case PageType::Unmapped: { + case PageType::Unmapped: LOG_ERROR(HW_Memory, "unmapped WriteBlock @ 0x%08X (start address = 0x%08X, size = %zu)", current_vaddr, dest_addr, size); break; - } + case PageType::Special: + if (WriteSpecialBlock(process, current_vaddr, src_buffer, copy_amount)) + break; + [[fallthrough]]; case PageType::Memory: { DEBUG_ASSERT(page_table.pointers[page_index]); @@ -544,26 +407,6 @@ void WriteBlock(const Kernel::Process& process, const VAddr dest_addr, const voi std::memcpy(dest_ptr, src_buffer, copy_amount); break; } - case PageType::Special: { - MMIORegionPointer handler = GetMMIOHandler(page_table, current_vaddr); - DEBUG_ASSERT(handler); - handler->WriteBlock(current_vaddr, src_buffer, copy_amount); - break; - } - case PageType::RasterizerCachedMemory: { - RasterizerFlushVirtualRegion(current_vaddr, static_cast(copy_amount), - FlushMode::FlushAndInvalidate); - std::memcpy(GetPointerFromVMA(process, current_vaddr), src_buffer, copy_amount); - break; - } - case PageType::RasterizerCachedSpecial: { - MMIORegionPointer handler = GetMMIOHandler(page_table, current_vaddr); - DEBUG_ASSERT(handler); - RasterizerFlushVirtualRegion(current_vaddr, static_cast(copy_amount), - FlushMode::FlushAndInvalidate); - handler->WriteBlock(current_vaddr, src_buffer, copy_amount); - break; - } default: UNREACHABLE(); } @@ -580,6 +423,8 @@ void WriteBlock(const VAddr dest_addr, const void* src_buffer, const size_t size } void ZeroBlock(const VAddr dest_addr, const size_t size) { + const auto& process = *Kernel::g_current_process; + size_t remaining_size = size; size_t page_index = dest_addr >> PAGE_BITS; size_t page_offset = dest_addr & PAGE_MASK; @@ -591,11 +436,14 @@ void ZeroBlock(const VAddr dest_addr, const size_t size) { const VAddr current_vaddr = static_cast((page_index << PAGE_BITS) + page_offset); switch (current_page_table->attributes[page_index]) { - case PageType::Unmapped: { + case PageType::Unmapped: LOG_ERROR(HW_Memory, "unmapped ZeroBlock @ 0x%08X (start address = 0x%08X, size = %zu)", current_vaddr, dest_addr, size); break; - } + case PageType::Special: + if (WriteSpecialBlock(process, current_vaddr, zeros.data(), copy_amount)) + break; + [[fallthrough]]; case PageType::Memory: { DEBUG_ASSERT(current_page_table->pointers[page_index]); @@ -603,25 +451,6 @@ void ZeroBlock(const VAddr dest_addr, const size_t size) { std::memset(dest_ptr, 0, copy_amount); break; } - case PageType::Special: { - DEBUG_ASSERT(GetMMIOHandler(current_vaddr)); - - GetMMIOHandler(current_vaddr)->WriteBlock(current_vaddr, zeros.data(), copy_amount); - break; - } - case PageType::RasterizerCachedMemory: { - RasterizerFlushVirtualRegion(current_vaddr, static_cast(copy_amount), - FlushMode::FlushAndInvalidate); - std::memset(GetPointerFromVMA(current_vaddr), 0, copy_amount); - break; - } - case PageType::RasterizerCachedSpecial: { - DEBUG_ASSERT(GetMMIOHandler(current_vaddr)); - RasterizerFlushVirtualRegion(current_vaddr, static_cast(copy_amount), - FlushMode::FlushAndInvalidate); - GetMMIOHandler(current_vaddr)->WriteBlock(current_vaddr, zeros.data(), copy_amount); - break; - } default: UNREACHABLE(); } @@ -633,6 +462,8 @@ void ZeroBlock(const VAddr dest_addr, const size_t size) { } void CopyBlock(VAddr dest_addr, VAddr src_addr, const size_t size) { + const auto& process = *Kernel::g_current_process; + size_t remaining_size = size; size_t page_index = src_addr >> PAGE_BITS; size_t page_offset = src_addr & PAGE_MASK; @@ -642,11 +473,18 @@ void CopyBlock(VAddr dest_addr, VAddr src_addr, const size_t size) { const VAddr current_vaddr = static_cast((page_index << PAGE_BITS) + page_offset); switch (current_page_table->attributes[page_index]) { - case PageType::Unmapped: { + case PageType::Unmapped: LOG_ERROR(HW_Memory, "unmapped CopyBlock @ 0x%08X (start address = 0x%08X, size = %zu)", current_vaddr, src_addr, size); ZeroBlock(dest_addr, copy_amount); break; + case PageType::Special: { + std::vector buffer(copy_amount); + if (ReadSpecialBlock(process, current_vaddr, buffer.data(), buffer.size())) { + WriteBlock(dest_addr, buffer.data(), buffer.size()); + break; + } + [[fallthrough]]; } case PageType::Memory: { DEBUG_ASSERT(current_page_table->pointers[page_index]); @@ -654,30 +492,6 @@ void CopyBlock(VAddr dest_addr, VAddr src_addr, const size_t size) { WriteBlock(dest_addr, src_ptr, copy_amount); break; } - case PageType::Special: { - DEBUG_ASSERT(GetMMIOHandler(current_vaddr)); - - std::vector buffer(copy_amount); - GetMMIOHandler(current_vaddr)->ReadBlock(current_vaddr, buffer.data(), buffer.size()); - WriteBlock(dest_addr, buffer.data(), buffer.size()); - break; - } - case PageType::RasterizerCachedMemory: { - RasterizerFlushVirtualRegion(current_vaddr, static_cast(copy_amount), - FlushMode::Flush); - WriteBlock(dest_addr, GetPointerFromVMA(current_vaddr), copy_amount); - break; - } - case PageType::RasterizerCachedSpecial: { - DEBUG_ASSERT(GetMMIOHandler(current_vaddr)); - RasterizerFlushVirtualRegion(current_vaddr, static_cast(copy_amount), - FlushMode::Flush); - - std::vector buffer(copy_amount); - GetMMIOHandler(current_vaddr)->ReadBlock(current_vaddr, buffer.data(), buffer.size()); - WriteBlock(dest_addr, buffer.data(), buffer.size()); - break; - } default: UNREACHABLE(); } @@ -691,43 +505,75 @@ void CopyBlock(VAddr dest_addr, VAddr src_addr, const size_t size) { } template <> -u8 ReadMMIO(MMIORegionPointer mmio_handler, VAddr addr) { - return mmio_handler->Read8(addr); +boost::optional ReadSpecial(VAddr addr) { + const PageTable& page_table = Kernel::g_current_process->vm_manager.page_table; + for (const auto& handler : GetSpecialHandlers(page_table, addr, sizeof(u8))) + if (auto result = handler->Read8(addr)) + return *result; + return {}; } template <> -u16 ReadMMIO(MMIORegionPointer mmio_handler, VAddr addr) { - return mmio_handler->Read16(addr); +boost::optional ReadSpecial(VAddr addr) { + const PageTable& page_table = Kernel::g_current_process->vm_manager.page_table; + for (const auto& handler : GetSpecialHandlers(page_table, addr, sizeof(u16))) + if (auto result = handler->Read16(addr)) + return *result; + return {}; } template <> -u32 ReadMMIO(MMIORegionPointer mmio_handler, VAddr addr) { - return mmio_handler->Read32(addr); +boost::optional ReadSpecial(VAddr addr) { + const PageTable& page_table = Kernel::g_current_process->vm_manager.page_table; + for (const auto& handler : GetSpecialHandlers(page_table, addr, sizeof(u32))) + if (auto result = handler->Read32(addr)) + return *result; + return {}; } template <> -u64 ReadMMIO(MMIORegionPointer mmio_handler, VAddr addr) { - return mmio_handler->Read64(addr); +boost::optional ReadSpecial(VAddr addr) { + const PageTable& page_table = Kernel::g_current_process->vm_manager.page_table; + for (const auto& handler : GetSpecialHandlers(page_table, addr, sizeof(u64))) + if (auto result = handler->Read64(addr)) + return *result; + return {}; } template <> -void WriteMMIO(MMIORegionPointer mmio_handler, VAddr addr, const u8 data) { - mmio_handler->Write8(addr, data); +bool WriteSpecial(VAddr addr, const u8 data) { + const PageTable& page_table = Kernel::g_current_process->vm_manager.page_table; + for (const auto& handler : GetSpecialHandlers(page_table, addr, sizeof(u8))) + if (handler->Write8(addr, data)) + return true; + return false; } template <> -void WriteMMIO(MMIORegionPointer mmio_handler, VAddr addr, const u16 data) { - mmio_handler->Write16(addr, data); +bool WriteSpecial(VAddr addr, const u16 data) { + const PageTable& page_table = Kernel::g_current_process->vm_manager.page_table; + for (const auto& handler : GetSpecialHandlers(page_table, addr, sizeof(u16))) + if (handler->Write16(addr, data)) + return true; + return false; } template <> -void WriteMMIO(MMIORegionPointer mmio_handler, VAddr addr, const u32 data) { - mmio_handler->Write32(addr, data); +bool WriteSpecial(VAddr addr, const u32 data) { + const PageTable& page_table = Kernel::g_current_process->vm_manager.page_table; + for (const auto& handler : GetSpecialHandlers(page_table, addr, sizeof(u32))) + if (handler->Write32(addr, data)) + return true; + return false; } template <> -void WriteMMIO(MMIORegionPointer mmio_handler, VAddr addr, const u64 data) { - mmio_handler->Write64(addr, data); +bool WriteSpecial(VAddr addr, const u64 data) { + const PageTable& page_table = Kernel::g_current_process->vm_manager.page_table; + for (const auto& handler : GetSpecialHandlers(page_table, addr, sizeof(u64))) + if (handler->Write64(addr, data)) + return true; + return false; } boost::optional TryVirtualToPhysicalAddress(const VAddr addr) { diff --git a/src/core/memory.h b/src/core/memory.h index 7e554f394..b2158ca46 100644 --- a/src/core/memory.h +++ b/src/core/memory.h @@ -8,10 +8,12 @@ #include #include #include +#include #include +#include #include #include "common/common_types.h" -#include "core/mmio.h" +#include "core/memory_hook.h" namespace Kernel { class Process; @@ -28,32 +30,35 @@ const u64 PAGE_SIZE = 1 << PAGE_BITS; const u64 PAGE_MASK = PAGE_SIZE - 1; const size_t PAGE_TABLE_NUM_ENTRIES = 1ULL << (36 - PAGE_BITS); -enum class PageType { +enum class PageType : u8 { /// Page is unmapped and should cause an access error. Unmapped, /// Page is mapped to regular memory. This is the only type you can get pointers to. Memory, - /// Page is mapped to regular memory, but also needs to check for rasterizer cache flushing and - /// invalidation - RasterizerCachedMemory, - /// Page is mapped to a I/O region. Writing and reading to this page is handled by functions. + /// Page is mapped to a memory hook, which intercepts read and write requests. Special, - /// Page is mapped to a I/O region, but also needs to check for rasterizer cache flushing and - /// invalidation - RasterizerCachedSpecial, }; struct SpecialRegion { - VAddr base; - u64 size; - MMIORegionPointer handler; + enum class Type { + DebugHook, + IODevice, + } type; + + MemoryHookPointer handler; + + bool operator<(const SpecialRegion& other) const { + return std::tie(type, handler) < std::tie(other.type, other.handler); + } + + bool operator==(const SpecialRegion& other) const { + return std::tie(type, handler) == std::tie(other.type, other.handler); + } }; /** * A (reasonably) fast way of allowing switchable and remappable process address spaces. It loosely - * mimics the way a real CPU page table works, but instead is optimized for minimal decoding and - * fetching requirements when accessing. In the usual case of an access to regular memory, it only - * requires an indexed fetch and a check for NULL. + * mimics the way a real CPU page table works. */ struct PageTable { /** @@ -66,19 +71,13 @@ struct PageTable { * Contains MMIO handlers that back memory regions whose entries in the `attribute` array is of * type `Special`. */ - std::vector special_regions; + boost::icl::interval_map> special_regions; /** * Array of fine grained page attributes. If it is set to any value other than `Memory`, then * the corresponding entry in `pointers` MUST be set to null. */ std::array attributes; - - /** - * Indicates the number of externally cached resources touching a page that should be - * flushed before the memory is accessed - */ - std::array cached_res_count; }; /// Physical memory regions as seen from the ARM11 @@ -243,33 +242,4 @@ boost::optional PhysicalToVirtualAddress(PAddr addr); */ u8* GetPhysicalPointer(PAddr address); -/** - * Adds the supplied value to the rasterizer resource cache counter of each - * page touching the region. - */ -void RasterizerMarkRegionCached(PAddr start, u64 size, int count_delta); - -/** - * Flushes any externally cached rasterizer resources touching the given region. - */ -void RasterizerFlushRegion(PAddr start, u64 size); - -/** - * Flushes and invalidates any externally cached rasterizer resources touching the given region. - */ -void RasterizerFlushAndInvalidateRegion(PAddr start, u64 size); - -enum class FlushMode { - /// Write back modified surfaces to RAM - Flush, - /// Write back modified surfaces to RAM, and also remove them from the cache - FlushAndInvalidate, -}; - -/** - * Flushes and invalidates any externally cached rasterizer resources touching the given virtual - * address region. - */ -void RasterizerFlushVirtualRegion(VAddr start, u64 size, FlushMode mode); - } // namespace Memory diff --git a/src/core/memory_hook.h b/src/core/memory_hook.h new file mode 100644 index 000000000..feebd850a --- /dev/null +++ b/src/core/memory_hook.h @@ -0,0 +1,46 @@ +// Copyright 2016 Citra Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#pragma once + +#include +#include +#include "common/common_types.h" + +namespace Memory { + +/** + * Memory hooks have two purposes: + * 1. To allow reads and writes to a region of memory to be intercepted. This is used to implement + * texture forwarding and memory breakpoints for debugging. + * 2. To allow for the implementation of MMIO devices. + * + * A hook may be mapped to multiple regions of memory. + * + * If a boost::none or false is returned from a function, the read/write request is passed through + * to the underlying memory region. + */ +class MemoryHook { +public: + virtual ~MemoryHook() = default; + + virtual boost::optional IsValidAddress(VAddr addr) = 0; + + virtual boost::optional Read8(VAddr addr) = 0; + virtual boost::optional Read16(VAddr addr) = 0; + virtual boost::optional Read32(VAddr addr) = 0; + virtual boost::optional Read64(VAddr addr) = 0; + + virtual bool ReadBlock(VAddr src_addr, void* dest_buffer, size_t size) = 0; + + virtual bool Write8(VAddr addr, u8 data) = 0; + virtual bool Write16(VAddr addr, u16 data) = 0; + virtual bool Write32(VAddr addr, u32 data) = 0; + virtual bool Write64(VAddr addr, u64 data) = 0; + + virtual bool WriteBlock(VAddr dest_addr, const void* src_buffer, size_t size) = 0; +}; + +using MemoryHookPointer = std::shared_ptr; +} // namespace Memory diff --git a/src/core/memory_setup.h b/src/core/memory_setup.h index 6f82a131e..9a1a4f4be 100644 --- a/src/core/memory_setup.h +++ b/src/core/memory_setup.h @@ -5,7 +5,7 @@ #pragma once #include "common/common_types.h" -#include "core/mmio.h" +#include "core/memory_hook.h" namespace Memory { @@ -26,7 +26,11 @@ void MapMemoryRegion(PageTable& page_table, VAddr base, u64 size, u8* target); * @param size The amount of bytes to map. Must be page-aligned. * @param mmio_handler The handler that backs the mapping. */ -void MapIoRegion(PageTable& page_table, VAddr base, u64 size, MMIORegionPointer mmio_handler); +void MapIoRegion(PageTable& page_table, VAddr base, u64 size, MemoryHookPointer mmio_handler); void UnmapRegion(PageTable& page_table, VAddr base, u64 size); + +void AddDebugHook(PageTable& page_table, VAddr base, u64 size, MemoryHookPointer hook); +void RemoveDebugHook(PageTable& page_table, VAddr base, u64 size, MemoryHookPointer hook); + } // namespace Memory diff --git a/src/core/mmio.h b/src/core/mmio.h deleted file mode 100644 index 5e3cc01af..000000000 --- a/src/core/mmio.h +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2016 Citra Emulator Project -// Licensed under GPLv2 or any later version -// Refer to the license.txt file included. - -#pragma once - -#include -#include "common/common_types.h" - -namespace Memory { - -/** - * Represents a device with memory mapped IO. - * A device may be mapped to multiple regions of memory. - */ -class MMIORegion { -public: - virtual ~MMIORegion() = default; - - virtual bool IsValidAddress(VAddr addr) = 0; - - virtual u8 Read8(VAddr addr) = 0; - virtual u16 Read16(VAddr addr) = 0; - virtual u32 Read32(VAddr addr) = 0; - virtual u64 Read64(VAddr addr) = 0; - - virtual bool ReadBlock(VAddr src_addr, void* dest_buffer, size_t size) = 0; - - virtual void Write8(VAddr addr, u8 data) = 0; - virtual void Write16(VAddr addr, u16 data) = 0; - virtual void Write32(VAddr addr, u32 data) = 0; - virtual void Write64(VAddr addr, u64 data) = 0; - - virtual bool WriteBlock(VAddr dest_addr, const void* src_buffer, size_t size) = 0; -}; - -using MMIORegionPointer = std::shared_ptr; -}; // namespace Memory diff --git a/src/tests/core/arm/arm_test_common.cpp b/src/tests/core/arm/arm_test_common.cpp index 2339bdfb8..88bbbc95c 100644 --- a/src/tests/core/arm/arm_test_common.cpp +++ b/src/tests/core/arm/arm_test_common.cpp @@ -19,8 +19,8 @@ TestEnvironment::TestEnvironment(bool mutable_memory_) page_table = &Kernel::g_current_process->vm_manager.page_table; page_table->pointers.fill(nullptr); + page_table->special_regions.clear(); page_table->attributes.fill(Memory::PageType::Unmapped); - page_table->cached_res_count.fill(0); Memory::MapIoRegion(*page_table, 0x00000000, 0x80000000, test_memory); Memory::MapIoRegion(*page_table, 0x80000000, 0x80000000, test_memory); @@ -62,11 +62,11 @@ void TestEnvironment::ClearWriteRecords() { TestEnvironment::TestMemory::~TestMemory() {} -bool TestEnvironment::TestMemory::IsValidAddress(VAddr addr) { +boost::optional TestEnvironment::TestMemory::IsValidAddress(VAddr addr) { return true; } -u8 TestEnvironment::TestMemory::Read8(VAddr addr) { +boost::optional TestEnvironment::TestMemory::Read8(VAddr addr) { auto iter = data.find(addr); if (iter == data.end()) { return addr; // Some arbitrary data @@ -74,16 +74,16 @@ u8 TestEnvironment::TestMemory::Read8(VAddr addr) { return iter->second; } -u16 TestEnvironment::TestMemory::Read16(VAddr addr) { - return Read8(addr) | static_cast(Read8(addr + 1)) << 8; +boost::optional TestEnvironment::TestMemory::Read16(VAddr addr) { + return *Read8(addr) | static_cast(*Read8(addr + 1)) << 8; } -u32 TestEnvironment::TestMemory::Read32(VAddr addr) { - return Read16(addr) | static_cast(Read16(addr + 2)) << 16; +boost::optional TestEnvironment::TestMemory::Read32(VAddr addr) { + return *Read16(addr) | static_cast(*Read16(addr + 2)) << 16; } -u64 TestEnvironment::TestMemory::Read64(VAddr addr) { - return Read32(addr) | static_cast(Read32(addr + 4)) << 32; +boost::optional TestEnvironment::TestMemory::Read64(VAddr addr) { + return *Read32(addr) | static_cast(*Read32(addr + 4)) << 32; } bool TestEnvironment::TestMemory::ReadBlock(VAddr src_addr, void* dest_buffer, size_t size) { @@ -91,34 +91,38 @@ bool TestEnvironment::TestMemory::ReadBlock(VAddr src_addr, void* dest_buffer, s u8* data = static_cast(dest_buffer); for (size_t i = 0; i < size; i++, addr++, data++) { - *data = Read8(addr); + *data = *Read8(addr); } return true; } -void TestEnvironment::TestMemory::Write8(VAddr addr, u8 data) { +bool TestEnvironment::TestMemory::Write8(VAddr addr, u8 data) { env->write_records.emplace_back(8, addr, data); if (env->mutable_memory) env->SetMemory8(addr, data); + return true; } -void TestEnvironment::TestMemory::Write16(VAddr addr, u16 data) { +bool TestEnvironment::TestMemory::Write16(VAddr addr, u16 data) { env->write_records.emplace_back(16, addr, data); if (env->mutable_memory) env->SetMemory16(addr, data); + return true; } -void TestEnvironment::TestMemory::Write32(VAddr addr, u32 data) { +bool TestEnvironment::TestMemory::Write32(VAddr addr, u32 data) { env->write_records.emplace_back(32, addr, data); if (env->mutable_memory) env->SetMemory32(addr, data); + return true; } -void TestEnvironment::TestMemory::Write64(VAddr addr, u64 data) { +bool TestEnvironment::TestMemory::Write64(VAddr addr, u64 data) { env->write_records.emplace_back(64, addr, data); if (env->mutable_memory) env->SetMemory64(addr, data); + return true; } bool TestEnvironment::TestMemory::WriteBlock(VAddr dest_addr, const void* src_buffer, size_t size) { diff --git a/src/tests/core/arm/arm_test_common.h b/src/tests/core/arm/arm_test_common.h index 592c28594..b66922d61 100644 --- a/src/tests/core/arm/arm_test_common.h +++ b/src/tests/core/arm/arm_test_common.h @@ -7,7 +7,7 @@ #include #include "common/common_types.h" -#include "core/mmio.h" +#include "core/memory_hook.h" namespace ArmTests { @@ -51,25 +51,25 @@ public: private: friend struct TestMemory; - struct TestMemory final : Memory::MMIORegion { + struct TestMemory final : Memory::MemoryHook { explicit TestMemory(TestEnvironment* env_) : env(env_) {} TestEnvironment* env; ~TestMemory() override; - bool IsValidAddress(VAddr addr) override; + boost::optional IsValidAddress(VAddr addr) override; - u8 Read8(VAddr addr) override; - u16 Read16(VAddr addr) override; - u32 Read32(VAddr addr) override; - u64 Read64(VAddr addr) override; + boost::optional Read8(VAddr addr) override; + boost::optional Read16(VAddr addr) override; + boost::optional Read32(VAddr addr) override; + boost::optional Read64(VAddr addr) override; bool ReadBlock(VAddr src_addr, void* dest_buffer, size_t size) override; - void Write8(VAddr addr, u8 data) override; - void Write16(VAddr addr, u16 data) override; - void Write32(VAddr addr, u32 data) override; - void Write64(VAddr addr, u64 data) override; + bool Write8(VAddr addr, u8 data) override; + bool Write16(VAddr addr, u16 data) override; + bool Write32(VAddr addr, u32 data) override; + bool Write64(VAddr addr, u64 data) override; bool WriteBlock(VAddr dest_addr, const void* src_buffer, size_t size) override; diff --git a/src/video_core/renderer_opengl/renderer_opengl.cpp b/src/video_core/renderer_opengl/renderer_opengl.cpp index 50396b5c1..8c23128ae 100644 --- a/src/video_core/renderer_opengl/renderer_opengl.cpp +++ b/src/video_core/renderer_opengl/renderer_opengl.cpp @@ -266,7 +266,7 @@ void RendererOpenGL::LoadFBToScreenInfo(const FramebufferInfo& framebuffer_info, screen_info.display_texture = screen_info.texture.resource.handle; screen_info.display_texcoords = MathUtil::Rectangle(0.f, 0.f, 1.f, 1.f); - Memory::RasterizerFlushRegion(framebuffer_info.address, size_in_bytes); + // Memory::RasterizerFlushRegion(framebuffer_info.address, size_in_bytes); state.texture_units[0].texture_2d = screen_info.texture.resource.handle; state.Apply(); -- cgit v1.2.3