diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/common/common_types.h | 1 | ||||
| -rw-r--r-- | src/core/device_memory.h | 16 | ||||
| -rw-r--r-- | src/core/device_memory_manager.h | 97 | ||||
| -rw-r--r-- | src/core/device_memory_manager.inc | 304 | ||||
| -rw-r--r-- | src/video_core/host1x/gpu_device_memory_manager.cpp | 21 | ||||
| -rw-r--r-- | src/video_core/host1x/gpu_device_memory_manager.h | 20 |
6 files changed, 459 insertions, 0 deletions
diff --git a/src/common/common_types.h b/src/common/common_types.h index 0fc225aff..ae04c4d60 100644 --- a/src/common/common_types.h +++ b/src/common/common_types.h @@ -45,6 +45,7 @@ using f32 = float; ///< 32-bit floating point using f64 = double; ///< 64-bit floating point using VAddr = u64; ///< Represents a pointer in the userspace virtual address space. +using DAddr = u64; ///< Represents a pointer in the device specific virtual address space. using PAddr = u64; ///< Represents a pointer in the ARM11 physical address space. using GPUVAddr = u64; ///< Represents a pointer in the GPU virtual address space. diff --git a/src/core/device_memory.h b/src/core/device_memory.h index 13388b73e..11bf0e326 100644 --- a/src/core/device_memory.h +++ b/src/core/device_memory.h @@ -32,6 +32,12 @@ public: } template <typename T> + PAddr GetRawPhysicalAddr(const T* ptr) const { + return static_cast<PAddr>(reinterpret_cast<uintptr_t>(ptr) - + reinterpret_cast<uintptr_t>(buffer.BackingBasePointer())); + } + + template <typename T> T* GetPointer(Common::PhysicalAddress addr) { return reinterpret_cast<T*>(buffer.BackingBasePointer() + (GetInteger(addr) - DramMemoryMap::Base)); @@ -43,6 +49,16 @@ public: (GetInteger(addr) - DramMemoryMap::Base)); } + template <typename T> + T* GetPointerFromRaw(PAddr addr) { + return reinterpret_cast<T*>(buffer.BackingBasePointer() + addr); + } + + template <typename T> + const T* GetPointerFromRaw(PAddr addr) const { + return reinterpret_cast<T*>(buffer.BackingBasePointer() + addr); + } + Common::HostMemory buffer; }; diff --git a/src/core/device_memory_manager.h b/src/core/device_memory_manager.h new file mode 100644 index 000000000..0861b792d --- /dev/null +++ b/src/core/device_memory_manager.h @@ -0,0 +1,97 @@ +// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#pragma once + +#include <deque> +#include <memory> + +#include "common/common_types.h" +#include "common/virtual_buffer.h" + +namespace Core { + +class DeviceMemory; + +namespace Memory { +class Memory; +} + +template <typename DTraits> +struct DeviceMemoryManagerAllocator; + +template <typename Traits> +class DeviceMemoryManager { + using DeviceInterface = typename Traits::DeviceInterface; + +public: + DeviceMemoryManager(const DeviceMemory& device_memory); + ~DeviceMemoryManager(); + + void BindInterface(DeviceInterface* interface); + + DAddr Allocate(size_t size); + void AllocateFixed(DAddr start, size_t size); + DAddr AllocatePinned(size_t size); + void Free(DAddr start, size_t size); + + void Map(DAddr address, VAddr virtual_address, size_t size, size_t p_id); + void Unmap(DAddr address, size_t size); + + // Write / Read + template <typename T> + T* GetPointer(DAddr address); + + template <typename T> + const T* GetPointer(DAddr address) const; + + template <typename T> + void Write(DAddr address, T value); + + template <typename T> + T Read(DAddr address) const; + + void ReadBlock(DAddr address, void* dest_pointer, size_t size); + void WriteBlock(DAddr address, void* src_pointer, size_t size); + + size_t RegisterProcess(Memory::Memory* memory); + void UnregisterProcess(size_t id); + +private: + static constexpr bool supports_pinning = Traits::supports_pinning; + static constexpr size_t device_virtual_bits = Traits::device_virtual_bits; + static constexpr size_t device_as_size = 1ULL << device_virtual_bits; + static constexpr size_t physical_max_bits = 33; + static constexpr size_t page_bits = 12; + static constexpr u32 physical_address_base = 1U << page_bits; + + template <typename T> + T* GetPointerFromRaw(PAddr addr) { + return reinterpret_cast<T*>(physical_base + addr); + } + + template <typename T> + const T* GetPointerFromRaw(PAddr addr) const { + return reinterpret_cast<T*>(physical_base + addr); + } + + template <typename T> + PAddr GetRawPhysicalAddr(const T* ptr) const { + return static_cast<PAddr>(reinterpret_cast<uintptr_t>(ptr) - physical_base); + } + + void WalkBlock(const DAddr addr, const std::size_t size, auto on_unmapped, auto on_memory, + auto increment); + + std::unique_ptr<DeviceMemoryManagerAllocator<Traits>> impl; + + const uintptr_t physical_base; + DeviceInterface* interface; + Common::VirtualBuffer<u32> compressed_physical_ptr; + Common::VirtualBuffer<u32> compressed_device_addr; + + std::deque<size_t> id_pool; + std::deque<Memory::Memory*> registered_processes; +}; + +} // namespace Core
\ No newline at end of file diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc new file mode 100644 index 000000000..1f52b92d5 --- /dev/null +++ b/src/core/device_memory_manager.inc @@ -0,0 +1,304 @@ +// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#include <atomic> +#include <memory> +#include <type_traits> + +#include "common/address_space.h" +#include "common/address_space.inc" +#include "common/alignment.h" +#include "common/scope_exit.h" +#include "core/device_memory.h" +#include "core/device_memory_manager.h" +#include "core/memory.h" + +namespace Core { + +struct EmptyAllocator { + EmptyAllocator([[maybe_unused]] DAddr address) {} +}; + +template <typename DTraits> +struct DeviceMemoryManagerAllocator { + static constexpr bool supports_pinning = DTraits::supports_pinning; + static constexpr size_t device_virtual_bits = DTraits::device_virtual_bits; + static constexpr size_t pin_bits = 32; + static constexpr DAddr first_address = 1ULL << Memory::YUZU_PAGEBITS; + static constexpr DAddr max_pin_area = supports_pinning ? 1ULL << pin_bits : first_address; + static constexpr DAddr max_device_area = 1ULL << device_virtual_bits; + + DeviceMemoryManagerAllocator() + : pin_allocator(first_address), + main_allocator(supports_pinning ? 1ULL << pin_bits : first_address) {} + + std::conditional_t<supports_pinning, Common::FlatAllocator<DAddr, 0, pin_bits>, EmptyAllocator> + pin_allocator; + Common::FlatAllocator<DAddr, 0, device_virtual_bits> main_allocator; + + /// Returns true when vaddr -> vaddr+size is fully contained in the buffer + template <bool pin_area> + [[nodiscard]] bool IsInBounds(VAddr addr, u64 size) const noexcept { + if constexpr (pin_area) { + return addr >= 0 && addr + size <= max_pin_area; + } else { + return addr >= max_pin_area && addr + size <= max_device_area; + } + } + + DAddr Allocate(size_t size) { + return main_allocator.Allocate(size); + } + + DAddr AllocatePinned(size_t size) { + return pin_allocator.Allocate(size); + } + + void DoInRange(DAddr address, size_t size, auto pin_func, auto main_func) { + if (IsInBounds<true>(address, size)) { + pin_func(address, size); + return; + } + if (IsInBounds<false>(address, size)) { + main_func(address, size); + return; + } + DAddr end_size = address + size - max_pin_area; + DAddr end_size2 = max_pin_area - address; + pin_func(address, end_size2); + main_func(max_pin_area, end_size); + } + + void AllocateFixed(DAddr b_address, size_t b_size) { + if constexpr (supports_pinning) { + DoInRange( + b_address, b_size, + [this](DAddr address, size_t size) { pin_allocator.AllocateFixed(address, size); }, + [this](DAddr address, size_t size) { + main_allocator.AllocateFixed(address, size); + }); + } else { + main_allocator.AllocateFixed(b_address, b_size); + } + } + + void Free(DAddr b_address, size_t b_size) { + if constexpr (supports_pinning) { + DoInRange( + b_address, b_size, + [this](DAddr address, size_t size) { pin_allocator.Free(address, size); }, + [this](DAddr address, size_t size) { main_allocator.Free(address, size); }); + } else { + main_allocator.Free(b_address, b_size); + } + } +}; + +template <typename Traits> +DeviceMemoryManager<Traits>::DeviceMemoryManager(const DeviceMemory& device_memory_) + : physical_base{reinterpret_cast<const uintptr_t>(device_memory_.buffer.BackingBasePointer())}, + interface{nullptr}, compressed_physical_ptr(device_as_size >> Memory::YUZU_PAGEBITS), + compressed_device_addr(1ULL << (physical_max_bits - Memory::YUZU_PAGEBITS)) { + impl = std::make_unique<DeviceMemoryManagerAllocator<Traits>>(); +} + +template <typename Traits> +DeviceMemoryManager<Traits>::~DeviceMemoryManager() = default; + +template <typename Traits> +void DeviceMemoryManager<Traits>::BindInterface(DeviceInterface* interface_) { + interface = interface_; +} + +template <typename Traits> +DAddr DeviceMemoryManager<Traits>::Allocate(size_t size) { + return impl->Allocate(size); +} + +template <typename Traits> +void DeviceMemoryManager<Traits>::AllocateFixed(DAddr start, size_t size) { + return impl->AllocateFixed(start, size); +} + +template <typename Traits> +DAddr DeviceMemoryManager<Traits>::AllocatePinned(size_t size) { + return impl->AllocatePinned(size); +} + +template <typename Traits> +void DeviceMemoryManager<Traits>::Free(DAddr start, size_t size) { + impl->Free(start, size); +} + +template <typename Traits> +void DeviceMemoryManager<Traits>::Map(DAddr address, VAddr virtual_address, size_t size, + size_t p_id) { + Core::Memory::Memory* process_memory = registered_processes[p_id]; + size_t start_page_d = address >> Memory::YUZU_PAGEBITS; + size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS; + std::atomic_thread_fence(std::memory_order_acquire); + for (size_t i = 0; i < num_pages; i++) { + auto* ptr = process_memory->GetPointer( + Common::ProcessAddress(virtual_address + i * Memory::YUZU_PAGESIZE)); + if (ptr == nullptr) [[unlikely]] { + compressed_physical_ptr[start_page_d + i] = 0; + continue; + } + auto phys_addr = static_cast<u32>(GetRawPhysicalAddr(ptr) >> Memory::YUZU_PAGEBITS) + 1U; + compressed_physical_ptr[start_page_d + i] = phys_addr; + compressed_device_addr[phys_addr - 1U] = static_cast<u32>(start_page_d + i); + } + std::atomic_thread_fence(std::memory_order_release); +} + +template <typename Traits> +void DeviceMemoryManager<Traits>::Unmap(DAddr address, size_t size) { + size_t start_page_d = address >> Memory::YUZU_PAGEBITS; + size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS; + std::atomic_thread_fence(std::memory_order_acquire); + for (size_t i = 0; i < num_pages; i++) { + auto phys_addr = compressed_physical_ptr[start_page_d + i]; + compressed_physical_ptr[start_page_d + i] = 0; + if (phys_addr != 0) { + compressed_device_addr[phys_addr - 1] = 0; + } + } + std::atomic_thread_fence(std::memory_order_release); +} + +template <typename Traits> +template <typename T> +T* DeviceMemoryManager<Traits>::GetPointer(DAddr address) { + const size_t index = address >> Memory::YUZU_PAGEBITS; + const size_t offset = address & Memory::YUZU_PAGEMASK; + auto phys_addr = compressed_physical_ptr[index]; + if (phys_addr == 0) [[unlikely]] { + return nullptr; + } + return GetPointerFromRaw<T>( + static_cast<PAddr>(((phys_addr - 1) << Memory::YUZU_PAGEBITS) + offset)); +} + +template <typename Traits> +template <typename T> +const T* DeviceMemoryManager<Traits>::GetPointer(DAddr address) const { + const size_t index = address >> Memory::YUZU_PAGEBITS; + const size_t offset = address & Memory::YUZU_PAGEMASK; + auto phys_addr = compressed_physical_ptr[index]; + if (phys_addr == 0) [[unlikely]] { + return nullptr; + } + return GetPointerFromRaw<T>( + static_cast<PAddr>(((phys_addr - 1) << Memory::YUZU_PAGEBITS) + offset)); +} + +template <typename Traits> +template <typename T> +void DeviceMemoryManager<Traits>::Write(DAddr address, T value) { + T* ptr = GetPointer<T>(address); + if (!ptr) [[unlikely]] { + return; + } + std::memcpy(ptr, &value, sizeof(T)); +} + +template <typename Traits> +template <typename T> +T DeviceMemoryManager<Traits>::Read(DAddr address) const { + const T* ptr = GetPointer<T>(address); + T result{}; + if (!ptr) [[unlikely]] { + return result; + } + std::memcpy(&result, ptr, sizeof(T)); + return result; +} + +template <typename Traits> +void DeviceMemoryManager<Traits>::WalkBlock(DAddr addr, std::size_t size, auto on_unmapped, + auto on_memory, auto increment) { + std::size_t remaining_size = size; + std::size_t page_index = addr >> Memory::YUZU_PAGEBITS; + std::size_t page_offset = addr & Memory::YUZU_PAGEMASK; + + while (remaining_size) { + const std::size_t copy_amount = + std::min(static_cast<std::size_t>(Memory::YUZU_PAGESIZE) - page_offset, remaining_size); + const auto current_vaddr = + static_cast<u64>((page_index << Memory::YUZU_PAGEBITS) + page_offset); + SCOPE_EXIT({ + page_index++; + page_offset = 0; + increment(copy_amount); + remaining_size -= copy_amount; + }); + + auto phys_addr = compressed_physical_ptr[page_index]; + if (phys_addr == 0) { + on_unmapped(copy_amount, current_vaddr); + continue; + } + auto* mem_ptr = GetPointerFromRaw<u8>( + static_cast<PAddr>(((phys_addr - 1) << Memory::YUZU_PAGEBITS) + page_offset)); + on_memory(copy_amount, mem_ptr); + } +} + +template <typename Traits> +void DeviceMemoryManager<Traits>::ReadBlock(DAddr address, void* dest_pointer, size_t size) { + WalkBlock( + address, size, + [&](size_t copy_amount, DAddr current_vaddr) { + LOG_ERROR( + HW_Memory, + "Unmapped Device ReadBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", + current_vaddr, address, size); + std::memset(dest_pointer, 0, copy_amount); + }, + [&](size_t copy_amount, const u8* const src_ptr) { + std::memcpy(dest_pointer, src_ptr, copy_amount); + }, + [&](const std::size_t copy_amount) { + dest_pointer = static_cast<u8*>(dest_pointer) + copy_amount; + }); +} + +template <typename Traits> +void DeviceMemoryManager<Traits>::WriteBlock(DAddr address, void* src_pointer, size_t size) { + WalkBlock( + address, size, + [&](size_t copy_amount, DAddr current_vaddr) { + LOG_ERROR( + HW_Memory, + "Unmapped Device WriteBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", + current_vaddr, address, size); + }, + [&](size_t copy_amount, u8* const dst_ptr) { + std::memcpy(dst_ptr, src_pointer, copy_amount); + }, + [&](const std::size_t copy_amount) { + src_pointer = static_cast<u8*>(src_pointer) + copy_amount; + }); +} + +template <typename Traits> +size_t DeviceMemoryManager<Traits>::RegisterProcess(Memory::Memory* memory_interface) { + size_t new_id; + if (!id_pool.empty()) { + new_id = id_pool.front(); + id_pool.pop_front(); + registered_processes[new_id] = memory_interface; + } else { + registered_processes.emplace_back(memory_interface); + new_id = registered_processes.size() - 1U; + } + return new_id; +} + +template <typename Traits> +void DeviceMemoryManager<Traits>::UnregisterProcess(size_t id) { + registered_processes[id] = nullptr; + id_pool.push_front(id); +} + +} // namespace Core
\ No newline at end of file diff --git a/src/video_core/host1x/gpu_device_memory_manager.cpp b/src/video_core/host1x/gpu_device_memory_manager.cpp new file mode 100644 index 000000000..2ca445081 --- /dev/null +++ b/src/video_core/host1x/gpu_device_memory_manager.cpp @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: 2023 yuzu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#include "core/device_memory_manager.inc" +#include "video_core/host1x/gpu_device_memory_manager.h" +#include "video_core/rasterizer_interface.h" + +template struct Core::DeviceMemoryManagerAllocator<Tegra::MaxwellDeviceTraits>; +template class Core::DeviceMemoryManager<Tegra::MaxwellDeviceTraits>; + +template const u8* Tegra::MaxwellDeviceMemoryManager::GetPointer<u8>(DAddr addr) const; +template u8* Tegra::MaxwellDeviceMemoryManager::GetPointer<u8>(DAddr addr); + +template u8 Tegra::MaxwellDeviceMemoryManager::Read<u8>(DAddr addr) const; +template u16 Tegra::MaxwellDeviceMemoryManager::Read<u16>(DAddr addr) const; +template u32 Tegra::MaxwellDeviceMemoryManager::Read<u32>(DAddr addr) const; +template u64 Tegra::MaxwellDeviceMemoryManager::Read<u64>(DAddr addr) const; +template void Tegra::MaxwellDeviceMemoryManager::Write<u8>(DAddr addr, u8 data); +template void Tegra::MaxwellDeviceMemoryManager::Write<u16>(DAddr addr, u16 data); +template void Tegra::MaxwellDeviceMemoryManager::Write<u32>(DAddr addr, u32 data); +template void Tegra::MaxwellDeviceMemoryManager::Write<u64>(DAddr addr, u64 data);
\ No newline at end of file diff --git a/src/video_core/host1x/gpu_device_memory_manager.h b/src/video_core/host1x/gpu_device_memory_manager.h new file mode 100644 index 000000000..30ad52017 --- /dev/null +++ b/src/video_core/host1x/gpu_device_memory_manager.h @@ -0,0 +1,20 @@ +// SPDX-FileCopyrightText: 2023 yuzu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#include "core/device_memory_manager.h" + +namespace VideoCore { +class RasterizerInterface; +} + +namespace Tegra { + +struct MaxwellDeviceTraits { + static constexpr bool supports_pinning = true; + static constexpr size_t device_virtual_bits = 34; + using DeviceInterface = typename VideoCore::RasterizerInterface; +}; + +using MaxwellDeviceMemoryManager = Core::DeviceMemoryManager<MaxwellDeviceTraits>; + +} // namespace Tegra
\ No newline at end of file |