8 files changed, 183 insertions, 58 deletions

diff --git a/src/video_core/texture_cache/descriptor_table.h b/src/video_core/texture_cache/descriptor_table.h
index ee4240288..1bad83fb4 100644
--- a/src/video_core/texture_cache/descriptor_table.h
+++ b/src/video_core/texture_cache/descriptor_table.h
@@ -19,9 +19,7 @@ public:
     explicit DescriptorTable(Tegra::MemoryManager& gpu_memory_) : gpu_memory{gpu_memory_} {}
 
     [[nodiscard]] bool Synchronize(GPUVAddr gpu_addr, u32 limit) {
-        [[likely]] if (current_gpu_addr == gpu_addr && current_limit == limit) {
-            return false;
-        }
+        [[likely]] if (current_gpu_addr == gpu_addr && current_limit == limit) { return false; }
         Refresh(gpu_addr, limit);
         return true;
     }
diff --git a/src/video_core/texture_cache/image_info.cpp b/src/video_core/texture_cache/image_info.cpp
index 852ec2519..e9100091e 100644
--- a/src/video_core/texture_cache/image_info.cpp
+++ b/src/video_core/texture_cache/image_info.cpp
@@ -100,6 +100,10 @@ ImageInfo::ImageInfo(const TICEntry& config) noexcept {
         ASSERT_MSG(false, "Invalid texture_type={}", static_cast<int>(config.texture_type.Value()));
         break;
     }
+    if (num_samples > 1) {
+        size.width *= NumSamplesX(config.msaa_mode);
+        size.height *= NumSamplesY(config.msaa_mode);
+    }
     if (type != ImageType::Linear) {
         // FIXME: Call this without passing *this
         layer_stride = CalculateLayerStride(*this);
diff --git a/src/video_core/texture_cache/samples_helper.h b/src/video_core/texture_cache/samples_helper.h
index d552bccf0..203ac1b11 100644
--- a/src/video_core/texture_cache/samples_helper.h
+++ b/src/video_core/texture_cache/samples_helper.h
@@ -51,4 +51,48 @@ namespace VideoCommon {
     return 1;
 }
 
+[[nodiscard]] inline int NumSamplesX(Tegra::Texture::MsaaMode msaa_mode) {
+    using Tegra::Texture::MsaaMode;
+    switch (msaa_mode) {
+    case MsaaMode::Msaa1x1:
+        return 1;
+    case MsaaMode::Msaa2x1:
+    case MsaaMode::Msaa2x1_D3D:
+    case MsaaMode::Msaa2x2:
+    case MsaaMode::Msaa2x2_VC4:
+    case MsaaMode::Msaa2x2_VC12:
+        return 2;
+    case MsaaMode::Msaa4x2:
+    case MsaaMode::Msaa4x2_D3D:
+    case MsaaMode::Msaa4x2_VC8:
+    case MsaaMode::Msaa4x2_VC24:
+    case MsaaMode::Msaa4x4:
+        return 4;
+    }
+    ASSERT_MSG(false, "Invalid MSAA mode={}", static_cast<int>(msaa_mode));
+    return 1;
+}
+
+[[nodiscard]] inline int NumSamplesY(Tegra::Texture::MsaaMode msaa_mode) {
+    using Tegra::Texture::MsaaMode;
+    switch (msaa_mode) {
+    case MsaaMode::Msaa1x1:
+    case MsaaMode::Msaa2x1:
+    case MsaaMode::Msaa2x1_D3D:
+        return 1;
+    case MsaaMode::Msaa2x2:
+    case MsaaMode::Msaa2x2_VC4:
+    case MsaaMode::Msaa2x2_VC12:
+    case MsaaMode::Msaa4x2:
+    case MsaaMode::Msaa4x2_D3D:
+    case MsaaMode::Msaa4x2_VC8:
+    case MsaaMode::Msaa4x2_VC24:
+        return 2;
+    case MsaaMode::Msaa4x4:
+        return 4;
+    }
+    ASSERT_MSG(false, "Invalid MSAA mode={}", static_cast<int>(msaa_mode));
+    return 1;
+}
+
 } // namespace VideoCommon
diff --git a/src/video_core/texture_cache/slot_vector.h b/src/video_core/texture_cache/slot_vector.h
index 1e2aad76a..9df6a2903 100644
--- a/src/video_core/texture_cache/slot_vector.h
+++ b/src/video_core/texture_cache/slot_vector.h
@@ -29,7 +29,7 @@ struct SlotId {
 };
 
 template <class T>
-requires std::is_nothrow_move_assignable_v<T> && std::is_nothrow_move_constructible_v<T>
+    requires std::is_nothrow_move_assignable_v<T> && std::is_nothrow_move_constructible_v<T>
 class SlotVector {
 public:
     class Iterator {
diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h
index 8e68a2e53..1b01990a4 100644
--- a/src/video_core/texture_cache/texture_cache.h
+++ b/src/video_core/texture_cache/texture_cache.h
@@ -39,6 +39,12 @@ TextureCache<P>::TextureCache(Runtime& runtime_, VideoCore::RasterizerInterface&
     sampler_descriptor.mipmap_filter.Assign(Tegra::Texture::TextureMipmapFilter::Linear);
     sampler_descriptor.cubemap_anisotropy.Assign(1);
 
+    // These values were chosen based on typical peak swizzle data sizes seen in some titles
+    static constexpr size_t SWIZZLE_DATA_BUFFER_INITIAL_CAPACITY = 8_MiB;
+    static constexpr size_t UNSWIZZLE_DATA_BUFFER_INITIAL_CAPACITY = 1_MiB;
+    swizzle_data_buffer.resize_destructive(SWIZZLE_DATA_BUFFER_INITIAL_CAPACITY);
+    unswizzle_data_buffer.resize_destructive(UNSWIZZLE_DATA_BUFFER_INITIAL_CAPACITY);
+
     // Make sure the first index is reserved for the null resources
     // This way the null resource becomes a compile time constant
     void(slot_images.insert(NullImageParams{}));
@@ -90,7 +96,8 @@ void TextureCache<P>::RunGarbageCollector() {
             const auto copies = FullDownloadCopies(image.info);
             image.DownloadMemory(map, copies);
             runtime.Finish();
-            SwizzleImage(*gpu_memory, image.gpu_addr, image.info, copies, map.mapped_span);
+            SwizzleImage(*gpu_memory, image.gpu_addr, image.info, copies, map.mapped_span,
+                         swizzle_data_buffer);
         }
         if (True(image.flags & ImageFlagBits::Tracked)) {
             UntrackImage(image, image_id);
@@ -142,6 +149,13 @@ typename P::ImageView& TextureCache<P>::GetImageView(ImageViewId id) noexcept {
 }
 
 template <class P>
+typename P::ImageView& TextureCache<P>::GetImageView(u32 index) noexcept {
+    const auto image_view_id = VisitImageView(channel_state->graphics_image_table,
+                                              channel_state->graphics_image_view_ids, index);
+    return slot_image_views[image_view_id];
+}
+
+template <class P>
 void TextureCache<P>::MarkModification(ImageId id) noexcept {
     MarkModification(slot_images[id]);
 }
@@ -461,7 +475,8 @@ void TextureCache<P>::DownloadMemory(VAddr cpu_addr, size_t size) {
         const auto copies = FullDownloadCopies(image.info);
         image.DownloadMemory(map, copies);
         runtime.Finish();
-        SwizzleImage(*gpu_memory, image.gpu_addr, image.info, copies, map.mapped_span);
+        SwizzleImage(*gpu_memory, image.gpu_addr, image.info, copies, map.mapped_span,
+                     swizzle_data_buffer);
     }
 }
 
@@ -638,7 +653,28 @@ bool TextureCache<P>::ShouldWaitAsyncFlushes() const noexcept {
 template <class P>
 void TextureCache<P>::CommitAsyncFlushes() {
     // This is intentionally passing the value by copy
-    committed_downloads.push(uncommitted_downloads);
+    if constexpr (IMPLEMENTS_ASYNC_DOWNLOADS) {
+        const std::span<const ImageId> download_ids = uncommitted_downloads;
+        if (download_ids.empty()) {
+            committed_downloads.emplace_back(std::move(uncommitted_downloads));
+            uncommitted_downloads.clear();
+            async_buffers.emplace_back(std::optional<AsyncBuffer>{});
+            return;
+        }
+        size_t total_size_bytes = 0;
+        for (const ImageId image_id : download_ids) {
+            total_size_bytes += slot_images[image_id].unswizzled_size_bytes;
+        }
+        auto download_map = runtime.DownloadStagingBuffer(total_size_bytes, true);
+        for (const ImageId image_id : download_ids) {
+            Image& image = slot_images[image_id];
+            const auto copies = FullDownloadCopies(image.info);
+            image.DownloadMemory(download_map, copies);
+            download_map.offset += Common::AlignUp(image.unswizzled_size_bytes, 64);
+        }
+        async_buffers.emplace_back(download_map);
+    }
+    committed_downloads.emplace_back(std::move(uncommitted_downloads));
     uncommitted_downloads.clear();
 }
 
@@ -647,36 +683,58 @@ void TextureCache<P>::PopAsyncFlushes() {
     if (committed_downloads.empty()) {
         return;
     }
-    const std::span<const ImageId> download_ids = committed_downloads.front();
-    if (download_ids.empty()) {
-        committed_downloads.pop();
-        return;
-    }
-    size_t total_size_bytes = 0;
-    for (const ImageId image_id : download_ids) {
-        total_size_bytes += slot_images[image_id].unswizzled_size_bytes;
-    }
-    auto download_map = runtime.DownloadStagingBuffer(total_size_bytes);
-    const size_t original_offset = download_map.offset;
-    for (const ImageId image_id : download_ids) {
-        Image& image = slot_images[image_id];
-        const auto copies = FullDownloadCopies(image.info);
-        image.DownloadMemory(download_map, copies);
-        download_map.offset += image.unswizzled_size_bytes;
-    }
-    // Wait for downloads to finish
-    runtime.Finish();
-
-    download_map.offset = original_offset;
-    std::span<u8> download_span = download_map.mapped_span;
-    for (const ImageId image_id : download_ids) {
-        const ImageBase& image = slot_images[image_id];
-        const auto copies = FullDownloadCopies(image.info);
-        SwizzleImage(*gpu_memory, image.gpu_addr, image.info, copies, download_span);
-        download_map.offset += image.unswizzled_size_bytes;
-        download_span = download_span.subspan(image.unswizzled_size_bytes);
+    if constexpr (IMPLEMENTS_ASYNC_DOWNLOADS) {
+        const std::span<const ImageId> download_ids = committed_downloads.front();
+        if (download_ids.empty()) {
+            committed_downloads.pop_front();
+            async_buffers.pop_front();
+            return;
+        }
+        auto download_map = *async_buffers.front();
+        std::span<u8> download_span = download_map.mapped_span;
+        for (size_t i = download_ids.size(); i > 0; i--) {
+            const ImageBase& image = slot_images[download_ids[i - 1]];
+            const auto copies = FullDownloadCopies(image.info);
+            download_map.offset -= Common::AlignUp(image.unswizzled_size_bytes, 64);
+            std::span<u8> download_span_alt = download_span.subspan(download_map.offset);
+            SwizzleImage(*gpu_memory, image.gpu_addr, image.info, copies, download_span_alt,
+                         swizzle_data_buffer);
+        }
+        runtime.FreeDeferredStagingBuffer(download_map);
+        committed_downloads.pop_front();
+        async_buffers.pop_front();
+    } else {
+        const std::span<const ImageId> download_ids = committed_downloads.front();
+        if (download_ids.empty()) {
+            committed_downloads.pop_front();
+            return;
+        }
+        size_t total_size_bytes = 0;
+        for (const ImageId image_id : download_ids) {
+            total_size_bytes += slot_images[image_id].unswizzled_size_bytes;
+        }
+        auto download_map = runtime.DownloadStagingBuffer(total_size_bytes);
+        const size_t original_offset = download_map.offset;
+        for (const ImageId image_id : download_ids) {
+            Image& image = slot_images[image_id];
+            const auto copies = FullDownloadCopies(image.info);
+            image.DownloadMemory(download_map, copies);
+            download_map.offset += image.unswizzled_size_bytes;
+        }
+        // Wait for downloads to finish
+        runtime.Finish();
+        download_map.offset = original_offset;
+        std::span<u8> download_span = download_map.mapped_span;
+        for (const ImageId image_id : download_ids) {
+            const ImageBase& image = slot_images[image_id];
+            const auto copies = FullDownloadCopies(image.info);
+            SwizzleImage(*gpu_memory, image.gpu_addr, image.info, copies, download_span,
+                         swizzle_data_buffer);
+            download_map.offset += image.unswizzled_size_bytes;
+            download_span = download_span.subspan(image.unswizzled_size_bytes);
+        }
+        committed_downloads.pop_front();
     }
-    committed_downloads.pop();
 }
 
 template <class P>
@@ -731,16 +789,25 @@ void TextureCache<P>::UploadImageContents(Image& image, StagingBuffer& staging)
     const GPUVAddr gpu_addr = image.gpu_addr;
 
     if (True(image.flags & ImageFlagBits::AcceleratedUpload)) {
-        gpu_memory->ReadBlockUnsafe(gpu_addr, mapped_span.data(), mapped_span.size_bytes());
+        gpu_memory->ReadBlock(gpu_addr, mapped_span.data(), mapped_span.size_bytes(),
+                              VideoCommon::CacheType::NoTextureCache);
         const auto uploads = FullUploadSwizzles(image.info);
         runtime.AccelerateImageUpload(image, staging, uploads);
-    } else if (True(image.flags & ImageFlagBits::Converted)) {
-        std::vector<u8> unswizzled_data(image.unswizzled_size_bytes);
-        auto copies = UnswizzleImage(*gpu_memory, gpu_addr, image.info, unswizzled_data);
-        ConvertImage(unswizzled_data, image.info, mapped_span, copies);
+        return;
+    }
+    const size_t guest_size_bytes = image.guest_size_bytes;
+    swizzle_data_buffer.resize_destructive(guest_size_bytes);
+    gpu_memory->ReadBlockUnsafe(gpu_addr, swizzle_data_buffer.data(), guest_size_bytes);
+
+    if (True(image.flags & ImageFlagBits::Converted)) {
+        unswizzle_data_buffer.resize_destructive(image.unswizzled_size_bytes);
+        auto copies = UnswizzleImage(*gpu_memory, gpu_addr, image.info, swizzle_data_buffer,
+                                     unswizzle_data_buffer);
+        ConvertImage(unswizzle_data_buffer, image.info, mapped_span, copies);
         image.UploadMemory(staging, copies);
     } else {
-        const auto copies = UnswizzleImage(*gpu_memory, gpu_addr, image.info, mapped_span);
+        const auto copies =
+            UnswizzleImage(*gpu_memory, gpu_addr, image.info, swizzle_data_buffer, mapped_span);
         image.UploadMemory(staging, copies);
     }
 }
@@ -910,7 +977,7 @@ void TextureCache<P>::InvalidateScale(Image& image) {
 }
 
 template <class P>
-u64 TextureCache<P>::GetScaledImageSizeBytes(ImageBase& image) {
+u64 TextureCache<P>::GetScaledImageSizeBytes(const ImageBase& image) {
     const u64 scale_up = static_cast<u64>(Settings::values.resolution_info.up_scale *
                                           Settings::values.resolution_info.up_scale);
     const u64 down_shift = static_cast<u64>(Settings::values.resolution_info.down_shift +
diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h
index 587339a31..485eaabaa 100644
--- a/src/video_core/texture_cache/texture_cache_base.h
+++ b/src/video_core/texture_cache/texture_cache_base.h
@@ -17,6 +17,7 @@
 #include "common/literals.h"
 #include "common/lru_cache.h"
 #include "common/polyfill_ranges.h"
+#include "common/scratch_buffer.h"
 #include "video_core/compatible_formats.h"
 #include "video_core/control/channel_state_cache.h"
 #include "video_core/delayed_destruction_ring.h"
@@ -91,6 +92,8 @@ class TextureCache : public VideoCommon::ChannelSetupCaches<TextureCacheChannelI
     static constexpr bool HAS_EMULATED_COPIES = P::HAS_EMULATED_COPIES;
     /// True when the API can provide info about the memory of the device.
     static constexpr bool HAS_DEVICE_MEMORY_INFO = P::HAS_DEVICE_MEMORY_INFO;
+    /// True when the API can do asynchronous texture downloads.
+    static constexpr bool IMPLEMENTS_ASYNC_DOWNLOADS = P::IMPLEMENTS_ASYNC_DOWNLOADS;
 
     static constexpr size_t UNSET_CHANNEL{std::numeric_limits<size_t>::max()};
 
@@ -105,6 +108,7 @@ class TextureCache : public VideoCommon::ChannelSetupCaches<TextureCacheChannelI
     using ImageView = typename P::ImageView;
     using Sampler = typename P::Sampler;
     using Framebuffer = typename P::Framebuffer;
+    using AsyncBuffer = typename P::AsyncBuffer;
 
     struct BlitImages {
         ImageId dst_id;
@@ -125,6 +129,9 @@ public:
     /// Return a reference to the given image view id
     [[nodiscard]] ImageView& GetImageView(ImageViewId id) noexcept;
 
+    /// Get the imageview from the graphics descriptor table in the specified index
+    [[nodiscard]] ImageView& GetImageView(u32 index) noexcept;
+
     /// Mark an image as modified from the GPU
     void MarkModification(ImageId id) noexcept;
 
@@ -202,7 +209,7 @@ public:
     /// Create channel state.
     void CreateChannel(Tegra::Control::ChannelState& channel) final override;
 
-    std::mutex mutex;
+    std::recursive_mutex mutex;
 
 private:
     /// Iterate over all page indices in a range
@@ -368,7 +375,7 @@ private:
     void InvalidateScale(Image& image);
     bool ScaleUp(Image& image);
     bool ScaleDown(Image& image);
-    u64 GetScaledImageSizeBytes(ImageBase& image);
+    u64 GetScaledImageSizeBytes(const ImageBase& image);
 
     Runtime& runtime;
 
@@ -402,7 +409,8 @@ private:
 
     // TODO: This data structure is not optimal and it should be reworked
     std::vector<ImageId> uncommitted_downloads;
-    std::queue<std::vector<ImageId>> committed_downloads;
+    std::deque<std::vector<ImageId>> committed_downloads;
+    std::deque<std::optional<AsyncBuffer>> async_buffers;
 
     struct LRUItemParams {
         using ObjectType = ImageId;
@@ -417,6 +425,9 @@ private:
 
     std::unordered_map<GPUVAddr, ImageAllocId> image_allocs_table;
 
+    Common::ScratchBuffer<u8> swizzle_data_buffer;
+    Common::ScratchBuffer<u8> unswizzle_data_buffer;
+
     u64 modification_tick = 0;
     u64 frame_tick = 0;
 };
diff --git a/src/video_core/texture_cache/util.cpp b/src/video_core/texture_cache/util.cpp
index e8c908b42..03acc68d9 100644
--- a/src/video_core/texture_cache/util.cpp
+++ b/src/video_core/texture_cache/util.cpp
@@ -505,7 +505,7 @@ void SwizzlePitchLinearImage(Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr
 
 void SwizzleBlockLinearImage(Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr,
                              const ImageInfo& info, const BufferImageCopy& copy,
-                             std::span<const u8> input) {
+                             std::span<const u8> input, Common::ScratchBuffer<u8>& tmp_buffer) {
     const Extent3D size = info.size;
     const LevelInfo level_info = MakeLevelInfo(info);
     const Extent2D tile_size = DefaultBlockSize(info.format);
@@ -534,8 +534,8 @@ void SwizzleBlockLinearImage(Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr
                        tile_size.height, info.tile_width_spacing);
     const size_t subresource_size = sizes[level];
 
-    const auto dst_data = std::make_unique<u8[]>(subresource_size);
-    const std::span<u8> dst(dst_data.get(), subresource_size);
+    tmp_buffer.resize_destructive(subresource_size);
+    const std::span<u8> dst(tmp_buffer);
 
     for (s32 layer = 0; layer < info.resources.layers; ++layer) {
         const std::span<const u8> src = input.subspan(host_offset);
@@ -765,8 +765,9 @@ bool IsValidEntry(const Tegra::MemoryManager& gpu_memory, const TICEntry& config
 }
 
 std::vector<BufferImageCopy> UnswizzleImage(Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr,
-                                            const ImageInfo& info, std::span<u8> output) {
-    const size_t guest_size_bytes = CalculateGuestSizeInBytes(info);
+                                            const ImageInfo& info, std::span<const u8> input,
+                                            std::span<u8> output) {
+    const size_t guest_size_bytes = input.size_bytes();
     const u32 bpp_log2 = BytesPerBlockLog2(info.format);
     const Extent3D size = info.size;
 
@@ -789,10 +790,6 @@ std::vector<BufferImageCopy> UnswizzleImage(Tegra::MemoryManager& gpu_memory, GP
             .image_extent = size,
         }};
     }
-    const auto input_data = std::make_unique<u8[]>(guest_size_bytes);
-    gpu_memory.ReadBlockUnsafe(gpu_addr, input_data.get(), guest_size_bytes);
-    const std::span<const u8> input(input_data.get(), guest_size_bytes);
-
     const LevelInfo level_info = MakeLevelInfo(info);
     const s32 num_layers = info.resources.layers;
     const s32 num_levels = info.resources.levels;
@@ -980,13 +977,14 @@ std::vector<SwizzleParameters> FullUploadSwizzles(const ImageInfo& info) {
 }
 
 void SwizzleImage(Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr, const ImageInfo& info,
-                  std::span<const BufferImageCopy> copies, std::span<const u8> memory) {
+                  std::span<const BufferImageCopy> copies, std::span<const u8> memory,
+                  Common::ScratchBuffer<u8>& tmp_buffer) {
     const bool is_pitch_linear = info.type == ImageType::Linear;
     for (const BufferImageCopy& copy : copies) {
         if (is_pitch_linear) {
             SwizzlePitchLinearImage(gpu_memory, gpu_addr, info, copy, memory);
         } else {
-            SwizzleBlockLinearImage(gpu_memory, gpu_addr, info, copy, memory);
+            SwizzleBlockLinearImage(gpu_memory, gpu_addr, info, copy, memory, tmp_buffer);
         }
     }
 }
diff --git a/src/video_core/texture_cache/util.h b/src/video_core/texture_cache/util.h
index 5e28f4ab3..d103db8ae 100644
--- a/src/video_core/texture_cache/util.h
+++ b/src/video_core/texture_cache/util.h
@@ -7,6 +7,7 @@
 #include <span>
 
 #include "common/common_types.h"
+#include "common/scratch_buffer.h"
 
 #include "video_core/surface.h"
 #include "video_core/texture_cache/image_base.h"
@@ -59,6 +60,7 @@ struct OverlapResult {
 
 [[nodiscard]] std::vector<BufferImageCopy> UnswizzleImage(Tegra::MemoryManager& gpu_memory,
                                                           GPUVAddr gpu_addr, const ImageInfo& info,
+                                                          std::span<const u8> input,
                                                           std::span<u8> output);
 
 [[nodiscard]] BufferCopy UploadBufferCopy(Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr,
@@ -76,7 +78,8 @@ void ConvertImage(std::span<const u8> input, const ImageInfo& info, std::span<u8
 [[nodiscard]] std::vector<SwizzleParameters> FullUploadSwizzles(const ImageInfo& info);
 
 void SwizzleImage(Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr, const ImageInfo& info,
-                  std::span<const BufferImageCopy> copies, std::span<const u8> memory);
+                  std::span<const BufferImageCopy> copies, std::span<const u8> memory,
+                  Common::ScratchBuffer<u8>& tmp_buffer);
 
 [[nodiscard]] bool IsBlockLinearSizeCompatible(const ImageInfo& new_info,
                                                const ImageInfo& overlap_info, u32 new_level,