From 241563d15c8b831a2d2b7c360d570cc721903d14 Mon Sep 17 00:00:00 2001 From: bunnei Date: Sun, 3 Mar 2019 23:17:35 -0500 Subject: [PATCH 1/9] gpu: Move GPUVAddr definition to common_types. --- src/common/common_types.h | 7 +++---- .../hle/service/nvdrv/devices/nvhost_as_gpu.cpp | 4 ++-- src/video_core/memory_manager.h | 3 --- src/video_core/renderer_opengl/gl_buffer_cache.cpp | 4 ++-- src/video_core/renderer_opengl/gl_buffer_cache.h | 2 +- src/video_core/renderer_opengl/gl_global_cache.cpp | 2 +- src/video_core/renderer_opengl/gl_global_cache.h | 2 +- .../renderer_opengl/gl_primitive_assembler.cpp | 3 +-- .../renderer_opengl/gl_primitive_assembler.h | 2 +- src/video_core/renderer_opengl/gl_rasterizer.cpp | 8 ++++---- .../renderer_opengl/gl_rasterizer_cache.cpp | 13 ++++++------- .../renderer_opengl/gl_rasterizer_cache.h | 6 +++--- src/video_core/renderer_opengl/gl_shader_cache.cpp | 4 ++-- src/video_core/renderer_vulkan/vk_buffer_cache.cpp | 3 +-- src/video_core/renderer_vulkan/vk_buffer_cache.h | 3 +-- src/yuzu/debugger/graphics/graphics_surface.cpp | 2 +- src/yuzu/debugger/graphics/graphics_surface.h | 2 +- 17 files changed, 31 insertions(+), 39 deletions(-) diff --git a/src/common/common_types.h b/src/common/common_types.h index 6b1766dca..4cec89fbd 100644 --- a/src/common/common_types.h +++ b/src/common/common_types.h @@ -40,10 +40,9 @@ using s64 = std::int64_t; ///< 64-bit signed int using f32 = float; ///< 32-bit floating point using f64 = double; ///< 64-bit floating point -// TODO: It would be nice to eventually replace these with strong types that prevent accidental -// conversion between each other. -using VAddr = u64; ///< Represents a pointer in the userspace virtual address space. -using PAddr = u64; ///< Represents a pointer in the ARM11 physical address space. +using VAddr = u64; ///< Represents a pointer in the userspace virtual address space. +using PAddr = u64; ///< Represents a pointer in the ARM11 physical address space. +using GPUVAddr = u64; ///< Represents a pointer in the GPU virtual address space. using u128 = std::array; static_assert(sizeof(u128) == 16, "u128 must be 128 bits wide"); diff --git a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp index b031ebc66..b7964d66e 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp @@ -89,7 +89,7 @@ u32 nvhost_as_gpu::Remap(const std::vector& input, std::vector& output) for (const auto& entry : entries) { LOG_WARNING(Service_NVDRV, "remap entry, offset=0x{:X} handle=0x{:X} pages=0x{:X}", entry.offset, entry.nvmap_handle, entry.pages); - Tegra::GPUVAddr offset = static_cast(entry.offset) << 0x10; + GPUVAddr offset = static_cast(entry.offset) << 0x10; auto object = nvmap_dev->GetObject(entry.nvmap_handle); if (!object) { LOG_CRITICAL(Service_NVDRV, "nvmap {} is an invalid handle!", entry.nvmap_handle); @@ -102,7 +102,7 @@ u32 nvhost_as_gpu::Remap(const std::vector& input, std::vector& output) u64 size = static_cast(entry.pages) << 0x10; ASSERT(size <= object->size); - Tegra::GPUVAddr returned = gpu.MemoryManager().MapBufferEx(object->addr, offset, size); + GPUVAddr returned = gpu.MemoryManager().MapBufferEx(object->addr, offset, size); ASSERT(returned == offset); } std::memcpy(output.data(), entries.data(), output.size()); diff --git a/src/video_core/memory_manager.h b/src/video_core/memory_manager.h index 425e2f31c..bb87fa24d 100644 --- a/src/video_core/memory_manager.h +++ b/src/video_core/memory_manager.h @@ -13,9 +13,6 @@ namespace Tegra { -/// Virtual addresses in the GPU's memory map are 64 bit. -using GPUVAddr = u64; - class MemoryManager final { public: MemoryManager(); diff --git a/src/video_core/renderer_opengl/gl_buffer_cache.cpp b/src/video_core/renderer_opengl/gl_buffer_cache.cpp index 5048ed6ce..f75c65825 100644 --- a/src/video_core/renderer_opengl/gl_buffer_cache.cpp +++ b/src/video_core/renderer_opengl/gl_buffer_cache.cpp @@ -21,8 +21,8 @@ CachedBufferEntry::CachedBufferEntry(VAddr cpu_addr, std::size_t size, GLintptr OGLBufferCache::OGLBufferCache(RasterizerOpenGL& rasterizer, std::size_t size) : RasterizerCache{rasterizer}, stream_buffer(size, true) {} -GLintptr OGLBufferCache::UploadMemory(Tegra::GPUVAddr gpu_addr, std::size_t size, - std::size_t alignment, bool cache) { +GLintptr OGLBufferCache::UploadMemory(GPUVAddr gpu_addr, std::size_t size, std::size_t alignment, + bool cache) { auto& memory_manager = Core::System::GetInstance().GPU().MemoryManager(); // Cache management is a big overhead, so only cache entries with a given size. diff --git a/src/video_core/renderer_opengl/gl_buffer_cache.h b/src/video_core/renderer_opengl/gl_buffer_cache.h index 1de1f84ae..fc33aa433 100644 --- a/src/video_core/renderer_opengl/gl_buffer_cache.h +++ b/src/video_core/renderer_opengl/gl_buffer_cache.h @@ -58,7 +58,7 @@ public: /// Uploads data from a guest GPU address. Returns host's buffer offset where it's been /// allocated. - GLintptr UploadMemory(Tegra::GPUVAddr gpu_addr, std::size_t size, std::size_t alignment = 4, + GLintptr UploadMemory(GPUVAddr gpu_addr, std::size_t size, std::size_t alignment = 4, bool cache = true); /// Uploads from a host memory. Returns host's buffer offset where it's been allocated. diff --git a/src/video_core/renderer_opengl/gl_global_cache.cpp b/src/video_core/renderer_opengl/gl_global_cache.cpp index c8dbcacbd..ac030cfc9 100644 --- a/src/video_core/renderer_opengl/gl_global_cache.cpp +++ b/src/video_core/renderer_opengl/gl_global_cache.cpp @@ -46,7 +46,7 @@ GlobalRegion GlobalRegionCacheOpenGL::TryGetReservedGlobalRegion(CacheAddr addr, return search->second; } -GlobalRegion GlobalRegionCacheOpenGL::GetUncachedGlobalRegion(Tegra::GPUVAddr addr, u32 size, +GlobalRegion GlobalRegionCacheOpenGL::GetUncachedGlobalRegion(GPUVAddr addr, u32 size, u8* host_ptr) { GlobalRegion region{TryGetReservedGlobalRegion(ToCacheAddr(host_ptr), size)}; if (!region) { diff --git a/src/video_core/renderer_opengl/gl_global_cache.h b/src/video_core/renderer_opengl/gl_global_cache.h index a840491f7..5a21ab66f 100644 --- a/src/video_core/renderer_opengl/gl_global_cache.h +++ b/src/video_core/renderer_opengl/gl_global_cache.h @@ -66,7 +66,7 @@ public: private: GlobalRegion TryGetReservedGlobalRegion(CacheAddr addr, u32 size) const; - GlobalRegion GetUncachedGlobalRegion(Tegra::GPUVAddr addr, u32 size, u8* host_ptr); + GlobalRegion GetUncachedGlobalRegion(GPUVAddr addr, u32 size, u8* host_ptr); void ReserveGlobalRegion(GlobalRegion region); std::unordered_map reserve; diff --git a/src/video_core/renderer_opengl/gl_primitive_assembler.cpp b/src/video_core/renderer_opengl/gl_primitive_assembler.cpp index 75d816795..2bcbd3da2 100644 --- a/src/video_core/renderer_opengl/gl_primitive_assembler.cpp +++ b/src/video_core/renderer_opengl/gl_primitive_assembler.cpp @@ -40,8 +40,7 @@ GLintptr PrimitiveAssembler::MakeQuadArray(u32 first, u32 count) { return index_offset; } -GLintptr PrimitiveAssembler::MakeQuadIndexed(Tegra::GPUVAddr gpu_addr, std::size_t index_size, - u32 count) { +GLintptr PrimitiveAssembler::MakeQuadIndexed(GPUVAddr gpu_addr, std::size_t index_size, u32 count) { const std::size_t map_size{CalculateQuadSize(count)}; auto [dst_pointer, index_offset] = buffer_cache.ReserveMemory(map_size); diff --git a/src/video_core/renderer_opengl/gl_primitive_assembler.h b/src/video_core/renderer_opengl/gl_primitive_assembler.h index a8cb88eb5..0e2e7dc36 100644 --- a/src/video_core/renderer_opengl/gl_primitive_assembler.h +++ b/src/video_core/renderer_opengl/gl_primitive_assembler.h @@ -24,7 +24,7 @@ public: GLintptr MakeQuadArray(u32 first, u32 count); - GLintptr MakeQuadIndexed(Tegra::GPUVAddr gpu_addr, std::size_t index_size, u32 count); + GLintptr MakeQuadIndexed(GPUVAddr gpu_addr, std::size_t index_size, u32 count); private: OGLBufferCache& buffer_cache; diff --git a/src/video_core/renderer_opengl/gl_rasterizer.cpp b/src/video_core/renderer_opengl/gl_rasterizer.cpp index 198c54872..e06dfe43f 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer.cpp +++ b/src/video_core/renderer_opengl/gl_rasterizer.cpp @@ -225,8 +225,8 @@ void RasterizerOpenGL::SetupVertexBuffer(GLuint vao) { if (!vertex_array.IsEnabled()) continue; - const Tegra::GPUVAddr start = vertex_array.StartAddress(); - const Tegra::GPUVAddr end = regs.vertex_array_limit[index].LimitAddress(); + const GPUVAddr start = vertex_array.StartAddress(); + const GPUVAddr end = regs.vertex_array_limit[index].LimitAddress(); ASSERT(end > start); const u64 size = end - start + 1; @@ -421,8 +421,8 @@ std::size_t RasterizerOpenGL::CalculateVertexArraysSize() const { if (!regs.vertex_array[index].IsEnabled()) continue; - const Tegra::GPUVAddr start = regs.vertex_array[index].StartAddress(); - const Tegra::GPUVAddr end = regs.vertex_array_limit[index].LimitAddress(); + const GPUVAddr start = regs.vertex_array[index].StartAddress(); + const GPUVAddr end = regs.vertex_array_limit[index].LimitAddress(); ASSERT(end > start); size += end - start + 1; diff --git a/src/video_core/renderer_opengl/gl_rasterizer_cache.cpp b/src/video_core/renderer_opengl/gl_rasterizer_cache.cpp index 57329cd61..1133fa1f9 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer_cache.cpp +++ b/src/video_core/renderer_opengl/gl_rasterizer_cache.cpp @@ -55,7 +55,7 @@ static void ApplyTextureDefaults(GLuint texture, u32 max_mip_level) { } } -void SurfaceParams::InitCacheParameters(Tegra::GPUVAddr gpu_addr_) { +void SurfaceParams::InitCacheParameters(GPUVAddr gpu_addr_) { auto& memory_manager{Core::System::GetInstance().GPU().MemoryManager()}; gpu_addr = gpu_addr_; @@ -222,7 +222,7 @@ std::size_t SurfaceParams::InnerMemorySize(bool force_gl, bool layer_only, } /*static*/ SurfaceParams SurfaceParams::CreateForDepthBuffer( - u32 zeta_width, u32 zeta_height, Tegra::GPUVAddr zeta_address, Tegra::DepthFormat format, + u32 zeta_width, u32 zeta_height, GPUVAddr zeta_address, Tegra::DepthFormat format, u32 block_width, u32 block_height, u32 block_depth, Tegra::Engines::Maxwell3D::Regs::InvMemoryLayout type) { SurfaceParams params{}; @@ -980,11 +980,11 @@ void RasterizerCacheOpenGL::FastLayeredCopySurface(const Surface& src_surface, const auto& init_params{src_surface->GetSurfaceParams()}; const auto& dst_params{dst_surface->GetSurfaceParams()}; auto& memory_manager{Core::System::GetInstance().GPU().MemoryManager()}; - Tegra::GPUVAddr address{init_params.gpu_addr}; + GPUVAddr address{init_params.gpu_addr}; const std::size_t layer_size{dst_params.LayerMemorySize()}; for (u32 layer = 0; layer < dst_params.depth; layer++) { for (u32 mipmap = 0; mipmap < dst_params.max_mip_level; mipmap++) { - const Tegra::GPUVAddr sub_address{address + dst_params.GetMipmapLevelOffset(mipmap)}; + const GPUVAddr sub_address{address + dst_params.GetMipmapLevelOffset(mipmap)}; const Surface& copy{TryGet(memory_manager.GetPointer(sub_address))}; if (!copy) { continue; @@ -1244,10 +1244,9 @@ static std::optional TryFindBestMipMap(std::size_t memory, const SurfacePar return {}; } -static std::optional TryFindBestLayer(Tegra::GPUVAddr addr, const SurfaceParams params, - u32 mipmap) { +static std::optional TryFindBestLayer(GPUVAddr addr, const SurfaceParams params, u32 mipmap) { const std::size_t size{params.LayerMemorySize()}; - Tegra::GPUVAddr start{params.gpu_addr + params.GetMipmapLevelOffset(mipmap)}; + GPUVAddr start{params.gpu_addr + params.GetMipmapLevelOffset(mipmap)}; for (u32 i = 0; i < params.depth; i++) { if (start == addr) { return {i}; diff --git a/src/video_core/renderer_opengl/gl_rasterizer_cache.h b/src/video_core/renderer_opengl/gl_rasterizer_cache.h index 9366f47f2..d76bc0ee7 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer_cache.h +++ b/src/video_core/renderer_opengl/gl_rasterizer_cache.h @@ -210,7 +210,7 @@ struct SurfaceParams { /// Creates SurfaceParams for a depth buffer configuration static SurfaceParams CreateForDepthBuffer( - u32 zeta_width, u32 zeta_height, Tegra::GPUVAddr zeta_address, Tegra::DepthFormat format, + u32 zeta_width, u32 zeta_height, GPUVAddr zeta_address, Tegra::DepthFormat format, u32 block_width, u32 block_height, u32 block_depth, Tegra::Engines::Maxwell3D::Regs::InvMemoryLayout type); @@ -232,7 +232,7 @@ struct SurfaceParams { } /// Initializes parameters for caching, should be called after everything has been initialized - void InitCacheParameters(Tegra::GPUVAddr gpu_addr); + void InitCacheParameters(GPUVAddr gpu_addr); std::string TargetName() const { switch (target) { @@ -297,7 +297,7 @@ struct SurfaceParams { bool srgb_conversion; // Parameters used for caching u8* host_ptr; - Tegra::GPUVAddr gpu_addr; + GPUVAddr gpu_addr; std::size_t size_in_bytes; std::size_t size_in_bytes_gl; diff --git a/src/video_core/renderer_opengl/gl_shader_cache.cpp b/src/video_core/renderer_opengl/gl_shader_cache.cpp index 1ed740877..1f8eca6f0 100644 --- a/src/video_core/renderer_opengl/gl_shader_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_cache.cpp @@ -32,7 +32,7 @@ struct UnspecializedShader { namespace { /// Gets the address for the specified shader stage program -Tegra::GPUVAddr GetShaderAddress(Maxwell::ShaderProgram program) { +GPUVAddr GetShaderAddress(Maxwell::ShaderProgram program) { const auto& gpu{Core::System::GetInstance().GPU().Maxwell3D()}; const auto& shader_config{gpu.regs.shader_config[static_cast(program)]}; return gpu.regs.code_address.CodeAddress() + shader_config.offset; @@ -486,7 +486,7 @@ Shader ShaderCacheOpenGL::GetStageProgram(Maxwell::ShaderProgram program) { } auto& memory_manager{Core::System::GetInstance().GPU().MemoryManager()}; - const Tegra::GPUVAddr program_addr{GetShaderAddress(program)}; + const GPUVAddr program_addr{GetShaderAddress(program)}; // Look up shader in the cache based on address const auto& host_ptr{memory_manager.GetPointer(program_addr)}; diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp index 95eab3fec..eac51ecb3 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp @@ -39,8 +39,7 @@ VKBufferCache::VKBufferCache(Tegra::MemoryManager& tegra_memory_manager, VKBufferCache::~VKBufferCache() = default; -u64 VKBufferCache::UploadMemory(Tegra::GPUVAddr gpu_addr, std::size_t size, u64 alignment, - bool cache) { +u64 VKBufferCache::UploadMemory(GPUVAddr gpu_addr, std::size_t size, u64 alignment, bool cache) { const auto cpu_addr{tegra_memory_manager.GpuToCpuAddress(gpu_addr)}; ASSERT_MSG(cpu_addr, "Invalid GPU address"); diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h index 8b415744b..08b786aad 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.h +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h @@ -68,8 +68,7 @@ public: /// Uploads data from a guest GPU address. Returns host's buffer offset where it's been /// allocated. - u64 UploadMemory(Tegra::GPUVAddr gpu_addr, std::size_t size, u64 alignment = 4, - bool cache = true); + u64 UploadMemory(GPUVAddr gpu_addr, std::size_t size, u64 alignment = 4, bool cache = true); /// Uploads from a host memory. Returns host's buffer offset where it's been allocated. u64 UploadHostMemory(const u8* raw_pointer, std::size_t size, u64 alignment = 4); diff --git a/src/yuzu/debugger/graphics/graphics_surface.cpp b/src/yuzu/debugger/graphics/graphics_surface.cpp index 29f01dfb2..11023ed63 100644 --- a/src/yuzu/debugger/graphics/graphics_surface.cpp +++ b/src/yuzu/debugger/graphics/graphics_surface.cpp @@ -261,7 +261,7 @@ void GraphicsSurfaceWidget::OnSurfaceSourceChanged(int new_value) { void GraphicsSurfaceWidget::OnSurfaceAddressChanged(qint64 new_value) { if (surface_address != new_value) { - surface_address = static_cast(new_value); + surface_address = static_cast(new_value); surface_source_list->setCurrentIndex(static_cast(Source::Custom)); emit Update(); diff --git a/src/yuzu/debugger/graphics/graphics_surface.h b/src/yuzu/debugger/graphics/graphics_surface.h index 323e39d94..89445b18f 100644 --- a/src/yuzu/debugger/graphics/graphics_surface.h +++ b/src/yuzu/debugger/graphics/graphics_surface.h @@ -87,7 +87,7 @@ private: QPushButton* save_surface; Source surface_source; - Tegra::GPUVAddr surface_address; + GPUVAddr surface_address; unsigned surface_width; unsigned surface_height; Tegra::Texture::TextureFormat surface_format; From 22d3dfbcd4c606d40e5ae36970db4661c302859f Mon Sep 17 00:00:00 2001 From: bunnei Date: Sun, 3 Mar 2019 23:54:16 -0500 Subject: [PATCH 2/9] gpu: Rewrite virtual memory manager using PageTable. --- src/common/page_table.cpp | 2 + src/common/page_table.h | 6 +- .../service/nvdrv/devices/nvhost_as_gpu.cpp | 12 +- src/video_core/dma_pusher.h | 1 - src/video_core/engines/kepler_memory.cpp | 2 +- src/video_core/engines/maxwell_3d.cpp | 8 +- src/video_core/gpu.cpp | 7 +- src/video_core/gpu.h | 6 +- src/video_core/memory_manager.cpp | 512 ++++++++++++------ src/video_core/memory_manager.h | 154 ++++-- src/video_core/rasterizer_interface.h | 1 - .../renderer_opengl/gl_global_cache.cpp | 4 +- .../renderer_opengl/gl_rasterizer_cache.cpp | 10 +- 13 files changed, 497 insertions(+), 228 deletions(-) diff --git a/src/common/page_table.cpp b/src/common/page_table.cpp index 8eba1c3f1..69b7abc54 100644 --- a/src/common/page_table.cpp +++ b/src/common/page_table.cpp @@ -16,6 +16,7 @@ void PageTable::Resize(std::size_t address_space_width_in_bits) { pointers.resize(num_page_table_entries); attributes.resize(num_page_table_entries); + backing_addr.resize(num_page_table_entries); // The default is a 39-bit address space, which causes an initial 1GB allocation size. If the // vector size is subsequently decreased (via resize), the vector might not automatically @@ -24,6 +25,7 @@ void PageTable::Resize(std::size_t address_space_width_in_bits) { pointers.shrink_to_fit(); attributes.shrink_to_fit(); + backing_addr.shrink_to_fit(); } } // namespace Common diff --git a/src/common/page_table.h b/src/common/page_table.h index 8339f2890..8b8ff0bb8 100644 --- a/src/common/page_table.h +++ b/src/common/page_table.h @@ -21,6 +21,8 @@ enum class PageType : u8 { RasterizerCachedMemory, /// Page is mapped to a I/O region. Writing and reading to this page is handled by functions. Special, + /// Page is allocated for use. + Allocated, }; struct SpecialRegion { @@ -66,7 +68,7 @@ struct PageTable { * Contains MMIO handlers that back memory regions whose entries in the `attribute` vector is * of type `Special`. */ - boost::icl::interval_map> special_regions; + boost::icl::interval_map> special_regions; /** * Vector of fine grained page attributes. If it is set to any value other than `Memory`, then @@ -74,6 +76,8 @@ struct PageTable { */ std::vector attributes; + std::vector backing_addr; + const std::size_t page_size_in_bits{}; }; diff --git a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp index b7964d66e..af62d33d2 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp @@ -173,16 +173,8 @@ u32 nvhost_as_gpu::UnmapBuffer(const std::vector& input, std::vector& ou return 0; } - auto& system_instance = Core::System::GetInstance(); - - // Remove this memory region from the rasterizer cache. - auto& gpu = system_instance.GPU(); - auto cpu_addr = gpu.MemoryManager().GpuToCpuAddress(params.offset); - ASSERT(cpu_addr); - gpu.FlushAndInvalidateRegion(ToCacheAddr(Memory::GetPointer(*cpu_addr)), itr->second.size); - - params.offset = gpu.MemoryManager().UnmapBuffer(params.offset, itr->second.size); - + params.offset = Core::System::GetInstance().GPU().MemoryManager().UnmapBuffer(params.offset, + itr->second.size); buffer_mappings.erase(itr->second.offset); std::memcpy(output.data(), ¶ms, output.size()); diff --git a/src/video_core/dma_pusher.h b/src/video_core/dma_pusher.h index 27a36348c..6ab06518f 100644 --- a/src/video_core/dma_pusher.h +++ b/src/video_core/dma_pusher.h @@ -9,7 +9,6 @@ #include "common/bit_field.h" #include "common/common_types.h" -#include "video_core/memory_manager.h" namespace Tegra { diff --git a/src/video_core/engines/kepler_memory.cpp b/src/video_core/engines/kepler_memory.cpp index 0931b9626..e259bf46b 100644 --- a/src/video_core/engines/kepler_memory.cpp +++ b/src/video_core/engines/kepler_memory.cpp @@ -46,7 +46,7 @@ void KeplerMemory::ProcessData(u32 data) { // contain a dirty surface that will have to be written back to memory. const GPUVAddr address{regs.dest.Address() + state.write_offset * sizeof(u32)}; rasterizer.InvalidateRegion(ToCacheAddr(memory_manager.GetPointer(address)), sizeof(u32)); - memory_manager.Write32(address, data); + memory_manager.Write(address, data); system.GPU().Maxwell3D().dirty_flags.OnMemoryWrite(); diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index c5d5be4ef..defcfbd3f 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -307,7 +307,7 @@ void Maxwell3D::ProcessQueryGet() { // Write the current query sequence to the sequence address. // TODO(Subv): Find out what happens if you use a long query type but mark it as a short // query. - memory_manager.Write32(sequence_address, sequence); + memory_manager.Write(sequence_address, sequence); } else { // Write the 128-bit result structure in long mode. Note: We emulate an infinitely fast // GPU, this command may actually take a while to complete in real hardware due to GPU @@ -395,7 +395,7 @@ void Maxwell3D::ProcessCBData(u32 value) { u8* ptr{memory_manager.GetPointer(address)}; rasterizer.InvalidateRegion(ToCacheAddr(ptr), sizeof(u32)); - memory_manager.Write32(address, value); + memory_manager.Write(address, value); dirty_flags.OnMemoryWrite(); @@ -447,7 +447,7 @@ std::vector Maxwell3D::GetStageTextures(Regs::ShaderSt for (GPUVAddr current_texture = tex_info_buffer.address + TextureInfoOffset; current_texture < tex_info_buffer_end; current_texture += sizeof(Texture::TextureHandle)) { - const Texture::TextureHandle tex_handle{memory_manager.Read32(current_texture)}; + const Texture::TextureHandle tex_handle{memory_manager.Read(current_texture)}; Texture::FullTextureInfo tex_info{}; // TODO(Subv): Use the shader to determine which textures are actually accessed. @@ -482,7 +482,7 @@ Texture::FullTextureInfo Maxwell3D::GetStageTexture(Regs::ShaderStage stage, ASSERT(tex_info_address < tex_info_buffer.address + tex_info_buffer.size); - const Texture::TextureHandle tex_handle{memory_manager.Read32(tex_info_address)}; + const Texture::TextureHandle tex_handle{memory_manager.Read(tex_info_address)}; Texture::FullTextureInfo tex_info{}; tex_info.index = static_cast(offset); diff --git a/src/video_core/gpu.cpp b/src/video_core/gpu.cpp index 66c690494..267a03f2d 100644 --- a/src/video_core/gpu.cpp +++ b/src/video_core/gpu.cpp @@ -12,6 +12,7 @@ #include "video_core/engines/maxwell_3d.h" #include "video_core/engines/maxwell_dma.h" #include "video_core/gpu.h" +#include "video_core/memory_manager.h" #include "video_core/renderer_base.h" namespace Tegra { @@ -287,7 +288,7 @@ void GPU::ProcessSemaphoreTriggerMethod() { block.timestamp = Core::System::GetInstance().CoreTiming().GetTicks(); memory_manager->WriteBlock(regs.smaphore_address.SmaphoreAddress(), &block, sizeof(block)); } else { - const u32 word{memory_manager->Read32(regs.smaphore_address.SmaphoreAddress())}; + const u32 word{memory_manager->Read(regs.smaphore_address.SmaphoreAddress())}; if ((op == GpuSemaphoreOperation::AcquireEqual && word == regs.semaphore_sequence) || (op == GpuSemaphoreOperation::AcquireGequal && static_cast(word - regs.semaphore_sequence) > 0) || @@ -314,11 +315,11 @@ void GPU::ProcessSemaphoreTriggerMethod() { } void GPU::ProcessSemaphoreRelease() { - memory_manager->Write32(regs.smaphore_address.SmaphoreAddress(), regs.semaphore_release); + memory_manager->Write(regs.smaphore_address.SmaphoreAddress(), regs.semaphore_release); } void GPU::ProcessSemaphoreAcquire() { - const u32 word = memory_manager->Read32(regs.smaphore_address.SmaphoreAddress()); + const u32 word = memory_manager->Read(regs.smaphore_address.SmaphoreAddress()); const auto value = regs.semaphore_acquire; if (word != value) { regs.acquire_active = true; diff --git a/src/video_core/gpu.h b/src/video_core/gpu.h index a14b95c30..c1830ac8d 100644 --- a/src/video_core/gpu.h +++ b/src/video_core/gpu.h @@ -9,7 +9,6 @@ #include "common/common_types.h" #include "core/hle/service/nvflinger/buffer_queue.h" #include "video_core/dma_pusher.h" -#include "video_core/memory_manager.h" using CacheAddr = std::uintptr_t; inline CacheAddr ToCacheAddr(const void* host_ptr) { @@ -124,6 +123,8 @@ enum class EngineID { MAXWELL_DMA_COPY_A = 0xB0B5, }; +class MemoryManager; + class GPU { public: explicit GPU(Core::System& system, VideoCore::RendererBase& renderer); @@ -244,9 +245,8 @@ protected: private: std::unique_ptr memory_manager; - /// Mapping of command subchannels to their bound engine ids. + /// Mapping of command subchannels to their bound engine ids std::array bound_engines = {}; - /// 3D engine std::unique_ptr maxwell_3d; /// 2D engine diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index 8e8f36f28..4c7faa067 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -5,198 +5,164 @@ #include "common/alignment.h" #include "common/assert.h" #include "common/logging/log.h" +#include "core/core.h" #include "core/memory.h" +#include "video_core/gpu.h" #include "video_core/memory_manager.h" +#include "video_core/rasterizer_interface.h" +#include "video_core/renderer_base.h" namespace Tegra { MemoryManager::MemoryManager() { - // Mark the first page as reserved, so that 0 is not a valid GPUVAddr. Otherwise, games might - // try to use 0 as a valid address, which is also used to mean nullptr. This fixes a bug with - // Undertale using 0 for a render target. - PageSlot(0) = static_cast(PageStatus::Reserved); + std::fill(page_table.pointers.begin(), page_table.pointers.end(), nullptr); + std::fill(page_table.attributes.begin(), page_table.attributes.end(), + Common::PageType::Unmapped); + page_table.Resize(address_space_width); + + // Initialize the map with a single free region covering the entire managed space. + VirtualMemoryArea initial_vma; + initial_vma.size = address_space_end; + vma_map.emplace(initial_vma.base, initial_vma); + + UpdatePageTableForVMA(initial_vma); } GPUVAddr MemoryManager::AllocateSpace(u64 size, u64 align) { - const std::optional gpu_addr{FindFreeBlock(0, size, align, PageStatus::Unmapped)}; - - ASSERT_MSG(gpu_addr, "unable to find available GPU memory"); - - for (u64 offset{}; offset < size; offset += PAGE_SIZE) { - VAddr& slot{PageSlot(*gpu_addr + offset)}; - - ASSERT(slot == static_cast(PageStatus::Unmapped)); - - slot = static_cast(PageStatus::Allocated); - } - - return *gpu_addr; -} - -GPUVAddr MemoryManager::AllocateSpace(GPUVAddr gpu_addr, u64 size, u64 align) { - for (u64 offset{}; offset < size; offset += PAGE_SIZE) { - VAddr& slot{PageSlot(gpu_addr + offset)}; - - ASSERT(slot == static_cast(PageStatus::Unmapped)); - - slot = static_cast(PageStatus::Allocated); - } - + const GPUVAddr gpu_addr{ + FindFreeRegion(address_space_base, size, align, VirtualMemoryArea::Type::Unmapped)}; + AllocateMemory(gpu_addr, 0, size); return gpu_addr; } -GPUVAddr MemoryManager::MapBufferEx(VAddr cpu_addr, u64 size) { - const std::optional gpu_addr{FindFreeBlock(0, size, PAGE_SIZE, PageStatus::Unmapped)}; - - ASSERT_MSG(gpu_addr, "unable to find available GPU memory"); - - for (u64 offset{}; offset < size; offset += PAGE_SIZE) { - VAddr& slot{PageSlot(*gpu_addr + offset)}; - - ASSERT(slot == static_cast(PageStatus::Unmapped)); - - slot = cpu_addr + offset; - } - - const MappedRegion region{cpu_addr, *gpu_addr, size}; - mapped_regions.push_back(region); - - return *gpu_addr; +GPUVAddr MemoryManager::AllocateSpace(GPUVAddr gpu_addr, u64 size, u64 align) { + AllocateMemory(gpu_addr, 0, size); + return gpu_addr; } -GPUVAddr MemoryManager::MapBufferEx(VAddr cpu_addr, GPUVAddr gpu_addr, u64 size) { - ASSERT((gpu_addr & PAGE_MASK) == 0); +GPUVAddr MemoryManager::MapBufferEx(GPUVAddr cpu_addr, u64 size) { + const GPUVAddr gpu_addr{ + FindFreeRegion(address_space_base, size, page_size, VirtualMemoryArea::Type::Unmapped)}; + MapBackingMemory(gpu_addr, Memory::GetPointer(cpu_addr), ((size + page_mask) & ~page_mask), + cpu_addr); + return gpu_addr; +} - if (PageSlot(gpu_addr) != static_cast(PageStatus::Allocated)) { - // Page has been already mapped. In this case, we must find a new area of memory to use that - // is different than the specified one. Super Mario Odyssey hits this scenario when changing - // areas, but we do not want to overwrite the old pages. - // TODO(bunnei): We need to write a hardware test to confirm this behavior. +GPUVAddr MemoryManager::MapBufferEx(GPUVAddr cpu_addr, GPUVAddr gpu_addr, u64 size) { + ASSERT((gpu_addr & page_mask) == 0); - LOG_ERROR(HW_GPU, "attempting to map addr 0x{:016X}, which is not available!", gpu_addr); - - const std::optional new_gpu_addr{ - FindFreeBlock(gpu_addr, size, PAGE_SIZE, PageStatus::Allocated)}; - - ASSERT_MSG(new_gpu_addr, "unable to find available GPU memory"); - - gpu_addr = *new_gpu_addr; - } - - for (u64 offset{}; offset < size; offset += PAGE_SIZE) { - VAddr& slot{PageSlot(gpu_addr + offset)}; - - ASSERT(slot == static_cast(PageStatus::Allocated)); - - slot = cpu_addr + offset; - } - - const MappedRegion region{cpu_addr, gpu_addr, size}; - mapped_regions.push_back(region); + MapBackingMemory(gpu_addr, Memory::GetPointer(cpu_addr), ((size + page_mask) & ~page_mask), + cpu_addr); return gpu_addr; } GPUVAddr MemoryManager::UnmapBuffer(GPUVAddr gpu_addr, u64 size) { - ASSERT((gpu_addr & PAGE_MASK) == 0); + ASSERT((gpu_addr & page_mask) == 0); - for (u64 offset{}; offset < size; offset += PAGE_SIZE) { - VAddr& slot{PageSlot(gpu_addr + offset)}; + const CacheAddr cache_addr{ToCacheAddr(GetPointer(gpu_addr))}; + Core::System::GetInstance().Renderer().Rasterizer().FlushAndInvalidateRegion(cache_addr, size); - ASSERT(slot != static_cast(PageStatus::Allocated) && - slot != static_cast(PageStatus::Unmapped)); + UnmapRange(gpu_addr, ((size + page_mask) & ~page_mask)); - slot = static_cast(PageStatus::Unmapped); - } - - // Delete the region mappings that are contained within the unmapped region - mapped_regions.erase(std::remove_if(mapped_regions.begin(), mapped_regions.end(), - [&](const MappedRegion& region) { - return region.gpu_addr <= gpu_addr && - region.gpu_addr + region.size < gpu_addr + size; - }), - mapped_regions.end()); return gpu_addr; } -GPUVAddr MemoryManager::GetRegionEnd(GPUVAddr region_start) const { - for (const auto& region : mapped_regions) { - const GPUVAddr region_end{region.gpu_addr + region.size}; - if (region_start >= region.gpu_addr && region_start < region_end) { - return region_end; - } - } - return {}; -} +GPUVAddr MemoryManager::FindFreeRegion(GPUVAddr region_start, u64 size, u64 align, + VirtualMemoryArea::Type vma_type) { -std::optional MemoryManager::FindFreeBlock(GPUVAddr region_start, u64 size, u64 align, - PageStatus status) { - GPUVAddr gpu_addr{region_start}; - u64 free_space{}; - align = (align + PAGE_MASK) & ~PAGE_MASK; + align = (align + page_mask) & ~page_mask; - while (gpu_addr + free_space < MAX_ADDRESS) { - if (PageSlot(gpu_addr + free_space) == static_cast(status)) { - free_space += PAGE_SIZE; - if (free_space >= size) { - return gpu_addr; - } - } else { - gpu_addr += free_space + PAGE_SIZE; - free_space = 0; - gpu_addr = Common::AlignUp(gpu_addr, align); - } - } + // Find the first Free VMA. + const GPUVAddr base = region_start; + const VMAHandle vma_handle = std::find_if(vma_map.begin(), vma_map.end(), [&](const auto& vma) { + if (vma.second.type != vma_type) + return false; - return {}; -} + const VAddr vma_end = vma.second.base + vma.second.size; + return vma_end > base && vma_end >= base + size; + }); -std::optional MemoryManager::GpuToCpuAddress(GPUVAddr gpu_addr) { - const VAddr base_addr{PageSlot(gpu_addr)}; - - if (base_addr == static_cast(PageStatus::Allocated) || - base_addr == static_cast(PageStatus::Unmapped) || - base_addr == static_cast(PageStatus::Reserved)) { + if (vma_handle == vma_map.end()) { return {}; } - return base_addr + (gpu_addr & PAGE_MASK); + return std::max(base, vma_handle->second.base); } -u8 MemoryManager::Read8(GPUVAddr addr) { - return Memory::Read8(*GpuToCpuAddress(addr)); +std::optional MemoryManager::GpuToCpuAddress(GPUVAddr gpu_addr) { + VAddr cpu_addr = page_table.backing_addr[gpu_addr >> page_bits]; + if (cpu_addr) { + return cpu_addr + (gpu_addr & page_mask); + } + + return {}; } -u16 MemoryManager::Read16(GPUVAddr addr) { - return Memory::Read16(*GpuToCpuAddress(addr)); +template +T MemoryManager::Read(GPUVAddr vaddr) { + const u8* page_pointer = page_table.pointers[vaddr >> page_bits]; + if (page_pointer) { + // NOTE: Avoid adding any extra logic to this fast-path block + T value; + std::memcpy(&value, &page_pointer[vaddr & page_mask], sizeof(T)); + return value; + } + + Common::PageType type = page_table.attributes[vaddr >> page_bits]; + switch (type) { + case Common::PageType::Unmapped: + LOG_ERROR(HW_GPU, "Unmapped Read{} @ 0x{:08X}", sizeof(T) * 8, vaddr); + return 0; + case Common::PageType::Memory: + ASSERT_MSG(false, "Mapped memory page without a pointer @ {:016X}", vaddr); + break; + default: + UNREACHABLE(); + } + return {}; } -u32 MemoryManager::Read32(GPUVAddr addr) { - return Memory::Read32(*GpuToCpuAddress(addr)); +template +void MemoryManager::Write(GPUVAddr vaddr, T data) { + u8* page_pointer = page_table.pointers[vaddr >> page_bits]; + if (page_pointer) { + // NOTE: Avoid adding any extra logic to this fast-path block + std::memcpy(&page_pointer[vaddr & page_mask], &data, sizeof(T)); + return; + } + + Common::PageType type = page_table.attributes[vaddr >> page_bits]; + switch (type) { + case Common::PageType::Unmapped: + LOG_ERROR(HW_GPU, "Unmapped Write{} 0x{:08X} @ 0x{:016X}", sizeof(data) * 8, + static_cast(data), vaddr); + return; + case Common::PageType::Memory: + ASSERT_MSG(false, "Mapped memory page without a pointer @ {:016X}", vaddr); + break; + default: + UNREACHABLE(); + } } -u64 MemoryManager::Read64(GPUVAddr addr) { - return Memory::Read64(*GpuToCpuAddress(addr)); -} - -void MemoryManager::Write8(GPUVAddr addr, u8 data) { - Memory::Write8(*GpuToCpuAddress(addr), data); -} - -void MemoryManager::Write16(GPUVAddr addr, u16 data) { - Memory::Write16(*GpuToCpuAddress(addr), data); -} - -void MemoryManager::Write32(GPUVAddr addr, u32 data) { - Memory::Write32(*GpuToCpuAddress(addr), data); -} - -void MemoryManager::Write64(GPUVAddr addr, u64 data) { - Memory::Write64(*GpuToCpuAddress(addr), data); -} +template u8 MemoryManager::Read(GPUVAddr addr); +template u16 MemoryManager::Read(GPUVAddr addr); +template u32 MemoryManager::Read(GPUVAddr addr); +template u64 MemoryManager::Read(GPUVAddr addr); +template void MemoryManager::Write(GPUVAddr addr, u8 data); +template void MemoryManager::Write(GPUVAddr addr, u16 data); +template void MemoryManager::Write(GPUVAddr addr, u32 data); +template void MemoryManager::Write(GPUVAddr addr, u64 data); u8* MemoryManager::GetPointer(GPUVAddr addr) { - return Memory::GetPointer(*GpuToCpuAddress(addr)); + u8* page_pointer = page_table.pointers[addr >> page_bits]; + if (page_pointer) { + return page_pointer + (addr & page_mask); + } + + LOG_ERROR(HW_GPU, "Unknown GetPointer @ 0x{:016X}", addr); + return {}; } void MemoryManager::ReadBlock(GPUVAddr src_addr, void* dest_buffer, std::size_t size) { @@ -210,13 +176,251 @@ void MemoryManager::CopyBlock(GPUVAddr dest_addr, GPUVAddr src_addr, std::size_t std::memcpy(GetPointer(dest_addr), GetPointer(src_addr), size); } -VAddr& MemoryManager::PageSlot(GPUVAddr gpu_addr) { - auto& block{page_table[(gpu_addr >> (PAGE_BITS + PAGE_TABLE_BITS)) & PAGE_TABLE_MASK]}; - if (!block) { - block = std::make_unique(); - block->fill(static_cast(PageStatus::Unmapped)); +void MemoryManager::MapPages(GPUVAddr base, u64 size, u8* memory, Common::PageType type, + VAddr backing_addr) { + LOG_DEBUG(HW_GPU, "Mapping {} onto {:016X}-{:016X}", fmt::ptr(memory), base * page_size, + (base + size) * page_size); + + VAddr end = base + size; + ASSERT_MSG(end <= page_table.pointers.size(), "out of range mapping at {:016X}", + base + page_table.pointers.size()); + + std::fill(page_table.attributes.begin() + base, page_table.attributes.begin() + end, type); + + if (memory == nullptr) { + std::fill(page_table.pointers.begin() + base, page_table.pointers.begin() + end, memory); + std::fill(page_table.backing_addr.begin() + base, page_table.backing_addr.begin() + end, + backing_addr); + } else { + while (base != end) { + page_table.pointers[base] = memory; + page_table.backing_addr[base] = backing_addr; + + base += 1; + memory += page_size; + backing_addr += page_size; + } + } +} + +void MemoryManager::MapMemoryRegion(GPUVAddr base, u64 size, u8* target, VAddr backing_addr) { + ASSERT_MSG((size & page_mask) == 0, "non-page aligned size: {:016X}", size); + ASSERT_MSG((base & page_mask) == 0, "non-page aligned base: {:016X}", base); + MapPages(base / page_size, size / page_size, target, Common::PageType::Memory, backing_addr); +} + +void MemoryManager::UnmapRegion(GPUVAddr base, u64 size) { + ASSERT_MSG((size & page_mask) == 0, "non-page aligned size: {:016X}", size); + ASSERT_MSG((base & page_mask) == 0, "non-page aligned base: {:016X}", base); + MapPages(base / page_size, size / page_size, nullptr, Common::PageType::Unmapped); +} + +bool VirtualMemoryArea::CanBeMergedWith(const VirtualMemoryArea& next) const { + ASSERT(base + size == next.base); + if (type != next.type) { + return {}; + } + if (type == VirtualMemoryArea::Type::Allocated && (offset + size != next.offset)) { + return {}; + } + if (type == VirtualMemoryArea::Type::Mapped && backing_memory + size != next.backing_memory) { + return {}; + } + return true; +} + +MemoryManager::VMAHandle MemoryManager::FindVMA(GPUVAddr target) const { + if (target >= address_space_end) { + return vma_map.end(); + } else { + return std::prev(vma_map.upper_bound(target)); + } +} + +MemoryManager::VMAHandle MemoryManager::AllocateMemory(GPUVAddr target, std::size_t offset, + u64 size) { + + // This is the appropriately sized VMA that will turn into our allocation. + VMAIter vma_handle = CarveVMA(target, size); + VirtualMemoryArea& final_vma = vma_handle->second; + ASSERT(final_vma.size == size); + + final_vma.type = VirtualMemoryArea::Type::Allocated; + final_vma.offset = offset; + UpdatePageTableForVMA(final_vma); + + return MergeAdjacent(vma_handle); +} + +MemoryManager::VMAHandle MemoryManager::MapBackingMemory(GPUVAddr target, u8* memory, u64 size, + VAddr backing_addr) { + // This is the appropriately sized VMA that will turn into our allocation. + VMAIter vma_handle = CarveVMA(target, size); + VirtualMemoryArea& final_vma = vma_handle->second; + ASSERT(final_vma.size == size); + + final_vma.type = VirtualMemoryArea::Type::Mapped; + final_vma.backing_memory = memory; + final_vma.backing_addr = backing_addr; + UpdatePageTableForVMA(final_vma); + + return MergeAdjacent(vma_handle); +} + +MemoryManager::VMAIter MemoryManager::Unmap(VMAIter vma_handle) { + VirtualMemoryArea& vma = vma_handle->second; + vma.type = VirtualMemoryArea::Type::Allocated; + vma.offset = 0; + vma.backing_memory = nullptr; + + UpdatePageTableForVMA(vma); + + return MergeAdjacent(vma_handle); +} + +void MemoryManager::UnmapRange(GPUVAddr target, u64 size) { + VMAIter vma = CarveVMARange(target, size); + const VAddr target_end = target + size; + + const VMAIter end = vma_map.end(); + // The comparison against the end of the range must be done using addresses since VMAs can be + // merged during this process, causing invalidation of the iterators. + while (vma != end && vma->second.base < target_end) { + vma = std::next(Unmap(vma)); + } + + ASSERT(FindVMA(target)->second.size >= size); +} + +MemoryManager::VMAIter MemoryManager::StripIterConstness(const VMAHandle& iter) { + // This uses a neat C++ trick to convert a const_iterator to a regular iterator, given + // non-const access to its container. + return vma_map.erase(iter, iter); // Erases an empty range of elements +} + +MemoryManager::VMAIter MemoryManager::CarveVMA(GPUVAddr base, u64 size) { + ASSERT_MSG((size & Tegra::MemoryManager::page_mask) == 0, "non-page aligned size: 0x{:016X}", + size); + ASSERT_MSG((base & Tegra::MemoryManager::page_mask) == 0, "non-page aligned base: 0x{:016X}", + base); + + VMAIter vma_handle = StripIterConstness(FindVMA(base)); + if (vma_handle == vma_map.end()) { + // Target address is outside the range managed by the kernel + return {}; + } + + const VirtualMemoryArea& vma = vma_handle->second; + if (vma.type == VirtualMemoryArea::Type::Mapped) { + // Region is already allocated + return {}; + } + + const VAddr start_in_vma = base - vma.base; + const VAddr end_in_vma = start_in_vma + size; + + if (end_in_vma < vma.size) { + // Split VMA at the end of the allocated region + SplitVMA(vma_handle, end_in_vma); + } + if (start_in_vma != 0) { + // Split VMA at the start of the allocated region + vma_handle = SplitVMA(vma_handle, start_in_vma); + } + + return vma_handle; +} + +MemoryManager::VMAIter MemoryManager::CarveVMARange(GPUVAddr target, u64 size) { + ASSERT_MSG((size & Tegra::MemoryManager::page_mask) == 0, "non-page aligned size: 0x{:016X}", + size); + ASSERT_MSG((target & Tegra::MemoryManager::page_mask) == 0, "non-page aligned base: 0x{:016X}", + target); + + const VAddr target_end = target + size; + ASSERT(target_end >= target); + ASSERT(size > 0); + + VMAIter begin_vma = StripIterConstness(FindVMA(target)); + const VMAIter i_end = vma_map.lower_bound(target_end); + if (std::any_of(begin_vma, i_end, [](const auto& entry) { + return entry.second.type == VirtualMemoryArea::Type::Unmapped; + })) { + return {}; + } + + if (target != begin_vma->second.base) { + begin_vma = SplitVMA(begin_vma, target - begin_vma->second.base); + } + + VMAIter end_vma = StripIterConstness(FindVMA(target_end)); + if (end_vma != vma_map.end() && target_end != end_vma->second.base) { + end_vma = SplitVMA(end_vma, target_end - end_vma->second.base); + } + + return begin_vma; +} + +MemoryManager::VMAIter MemoryManager::SplitVMA(VMAIter vma_handle, u64 offset_in_vma) { + VirtualMemoryArea& old_vma = vma_handle->second; + VirtualMemoryArea new_vma = old_vma; // Make a copy of the VMA + + // For now, don't allow no-op VMA splits (trying to split at a boundary) because it's probably + // a bug. This restriction might be removed later. + ASSERT(offset_in_vma < old_vma.size); + ASSERT(offset_in_vma > 0); + + old_vma.size = offset_in_vma; + new_vma.base += offset_in_vma; + new_vma.size -= offset_in_vma; + + switch (new_vma.type) { + case VirtualMemoryArea::Type::Unmapped: + break; + case VirtualMemoryArea::Type::Allocated: + new_vma.offset += offset_in_vma; + break; + case VirtualMemoryArea::Type::Mapped: + new_vma.backing_memory += offset_in_vma; + break; + } + + ASSERT(old_vma.CanBeMergedWith(new_vma)); + + return vma_map.emplace_hint(std::next(vma_handle), new_vma.base, new_vma); +} + +MemoryManager::VMAIter MemoryManager::MergeAdjacent(VMAIter iter) { + const VMAIter next_vma = std::next(iter); + if (next_vma != vma_map.end() && iter->second.CanBeMergedWith(next_vma->second)) { + iter->second.size += next_vma->second.size; + vma_map.erase(next_vma); + } + + if (iter != vma_map.begin()) { + VMAIter prev_vma = std::prev(iter); + if (prev_vma->second.CanBeMergedWith(iter->second)) { + prev_vma->second.size += iter->second.size; + vma_map.erase(iter); + iter = prev_vma; + } + } + + return iter; +} + +void MemoryManager::UpdatePageTableForVMA(const VirtualMemoryArea& vma) { + switch (vma.type) { + case VirtualMemoryArea::Type::Unmapped: + UnmapRegion(vma.base, vma.size); + break; + case VirtualMemoryArea::Type::Allocated: + MapMemoryRegion(vma.base, vma.size, nullptr, vma.backing_addr); + break; + case VirtualMemoryArea::Type::Mapped: + MapMemoryRegion(vma.base, vma.size, vma.backing_memory, vma.backing_addr); + break; } - return (*block)[(gpu_addr >> PAGE_BITS) & PAGE_BLOCK_MASK]; } } // namespace Tegra diff --git a/src/video_core/memory_manager.h b/src/video_core/memory_manager.h index bb87fa24d..ac1b42936 100644 --- a/src/video_core/memory_manager.h +++ b/src/video_core/memory_manager.h @@ -1,79 +1,147 @@ -// Copyright 2018 yuzu emulator team +// Copyright 2018 yuzu emulator team // Licensed under GPLv2 or any later version // Refer to the license.txt file included. #pragma once -#include -#include +#include #include -#include #include "common/common_types.h" +#include "common/page_table.h" namespace Tegra { +/** + * Represents a VMA in an address space. A VMA is a contiguous region of virtual addressing space + * with homogeneous attributes across its extents. In this particular implementation each VMA is + * also backed by a single host memory allocation. + */ +struct VirtualMemoryArea { + enum class Type : u8 { + Unmapped, + Allocated, + Mapped, + }; + + /// Virtual base address of the region. + GPUVAddr base{}; + /// Size of the region. + u64 size{}; + /// Memory area mapping type. + Type type{Type::Unmapped}; + /// CPU memory mapped address corresponding to this memory area. + VAddr backing_addr{}; + /// Offset into the backing_memory the mapping starts from. + std::size_t offset{}; + /// Pointer backing this VMA. + u8* backing_memory{}; + + /// Tests if this area can be merged to the right with `next`. + bool CanBeMergedWith(const VirtualMemoryArea& next) const; +}; + class MemoryManager final { public: MemoryManager(); GPUVAddr AllocateSpace(u64 size, u64 align); GPUVAddr AllocateSpace(GPUVAddr gpu_addr, u64 size, u64 align); - GPUVAddr MapBufferEx(VAddr cpu_addr, u64 size); - GPUVAddr MapBufferEx(VAddr cpu_addr, GPUVAddr gpu_addr, u64 size); + GPUVAddr MapBufferEx(GPUVAddr cpu_addr, u64 size); + GPUVAddr MapBufferEx(GPUVAddr cpu_addr, GPUVAddr gpu_addr, u64 size); GPUVAddr UnmapBuffer(GPUVAddr gpu_addr, u64 size); - GPUVAddr GetRegionEnd(GPUVAddr region_start) const; std::optional GpuToCpuAddress(GPUVAddr gpu_addr); - static constexpr u64 PAGE_BITS = 16; - static constexpr u64 PAGE_SIZE = 1 << PAGE_BITS; - static constexpr u64 PAGE_MASK = PAGE_SIZE - 1; + template + T Read(GPUVAddr vaddr); - u8 Read8(GPUVAddr addr); - u16 Read16(GPUVAddr addr); - u32 Read32(GPUVAddr addr); - u64 Read64(GPUVAddr addr); - - void Write8(GPUVAddr addr, u8 data); - void Write16(GPUVAddr addr, u16 data); - void Write32(GPUVAddr addr, u32 data); - void Write64(GPUVAddr addr, u64 data); + template + void Write(GPUVAddr vaddr, T data); u8* GetPointer(GPUVAddr vaddr); void ReadBlock(GPUVAddr src_addr, void* dest_buffer, std::size_t size); void WriteBlock(GPUVAddr dest_addr, const void* src_buffer, std::size_t size); - void CopyBlock(VAddr dest_addr, VAddr src_addr, std::size_t size); + void CopyBlock(GPUVAddr dest_addr, GPUVAddr src_addr, std::size_t size); private: - enum class PageStatus : u64 { - Unmapped = 0xFFFFFFFFFFFFFFFFULL, - Allocated = 0xFFFFFFFFFFFFFFFEULL, - Reserved = 0xFFFFFFFFFFFFFFFDULL, - }; + using VMAMap = std::map; + using VMAHandle = VMAMap::const_iterator; + using VMAIter = VMAMap::iterator; - std::optional FindFreeBlock(GPUVAddr region_start, u64 size, u64 align, - PageStatus status); - VAddr& PageSlot(GPUVAddr gpu_addr); + void MapPages(GPUVAddr base, u64 size, u8* memory, Common::PageType type, + VAddr backing_addr = 0); + void MapMemoryRegion(GPUVAddr base, u64 size, u8* target, VAddr backing_addr); + void UnmapRegion(GPUVAddr base, u64 size); - static constexpr u64 MAX_ADDRESS{0x10000000000ULL}; - static constexpr u64 PAGE_TABLE_BITS{10}; - static constexpr u64 PAGE_TABLE_SIZE{1 << PAGE_TABLE_BITS}; - static constexpr u64 PAGE_TABLE_MASK{PAGE_TABLE_SIZE - 1}; - static constexpr u64 PAGE_BLOCK_BITS{14}; - static constexpr u64 PAGE_BLOCK_SIZE{1 << PAGE_BLOCK_BITS}; - static constexpr u64 PAGE_BLOCK_MASK{PAGE_BLOCK_SIZE - 1}; + /// Finds the VMA in which the given address is included in, or `vma_map.end()`. + VMAHandle FindVMA(GPUVAddr target) const; - using PageBlock = std::array; - std::array, PAGE_TABLE_SIZE> page_table{}; + VMAHandle AllocateMemory(GPUVAddr target, std::size_t offset, u64 size); - struct MappedRegion { - VAddr cpu_addr; - GPUVAddr gpu_addr; - u64 size; - }; + /** + * Maps an unmanaged host memory pointer at a given address. + * + * @param target The guest address to start the mapping at. + * @param memory The memory to be mapped. + * @param size Size of the mapping. + * @param state MemoryState tag to attach to the VMA. + */ + VMAHandle MapBackingMemory(GPUVAddr target, u8* memory, u64 size, VAddr backing_addr); - std::vector mapped_regions; + /// Unmaps a range of addresses, splitting VMAs as necessary. + void UnmapRange(GPUVAddr target, u64 size); + + /// Converts a VMAHandle to a mutable VMAIter. + VMAIter StripIterConstness(const VMAHandle& iter); + + /// Unmaps the given VMA. + VMAIter Unmap(VMAIter vma); + + /** + * Carves a VMA of a specific size at the specified address by splitting Free VMAs while doing + * the appropriate error checking. + */ + VMAIter CarveVMA(GPUVAddr base, u64 size); + + /** + * Splits the edges of the given range of non-Free VMAs so that there is a VMA split at each + * end of the range. + */ + VMAIter CarveVMARange(GPUVAddr base, u64 size); + + /** + * Splits a VMA in two, at the specified offset. + * @returns the right side of the split, with the original iterator becoming the left side. + */ + VMAIter SplitVMA(VMAIter vma, u64 offset_in_vma); + + /** + * Checks for and merges the specified VMA with adjacent ones if possible. + * @returns the merged VMA or the original if no merging was possible. + */ + VMAIter MergeAdjacent(VMAIter vma); + + /// Updates the pages corresponding to this VMA so they match the VMA's attributes. + void UpdatePageTableForVMA(const VirtualMemoryArea& vma); + + GPUVAddr FindFreeRegion(GPUVAddr region_start, u64 size, u64 align, + VirtualMemoryArea::Type vma_type); + +private: + static constexpr u64 page_bits{16}; + static constexpr u64 page_size{1 << page_bits}; + static constexpr u64 page_mask{page_size - 1}; + + /// Address space in bits, this is fairly arbitrary but sufficiently large. + static constexpr u32 address_space_width = 39; + /// Start address for mapping, this is fairly arbitrary but must be non-zero. + static constexpr GPUVAddr address_space_base = 0x100000; + /// End of address space, based on address space in bits. + static constexpr GPUVAddr address_space_end = 1ULL << address_space_width; + + Common::PageTable page_table{page_bits}; + VMAMap vma_map; }; } // namespace Tegra diff --git a/src/video_core/rasterizer_interface.h b/src/video_core/rasterizer_interface.h index 76e292e87..d7b86df38 100644 --- a/src/video_core/rasterizer_interface.h +++ b/src/video_core/rasterizer_interface.h @@ -9,7 +9,6 @@ #include "common/common_types.h" #include "video_core/engines/fermi_2d.h" #include "video_core/gpu.h" -#include "video_core/memory_manager.h" namespace VideoCore { diff --git a/src/video_core/renderer_opengl/gl_global_cache.cpp b/src/video_core/renderer_opengl/gl_global_cache.cpp index ac030cfc9..0fbfbad55 100644 --- a/src/video_core/renderer_opengl/gl_global_cache.cpp +++ b/src/video_core/renderer_opengl/gl_global_cache.cpp @@ -76,8 +76,8 @@ GlobalRegion GlobalRegionCacheOpenGL::GetGlobalRegion( const auto cbufs{gpu.Maxwell3D().state.shader_stages[static_cast(stage)]}; const auto addr{cbufs.const_buffers[global_region.GetCbufIndex()].address + global_region.GetCbufOffset()}; - const auto actual_addr{memory_manager.Read64(addr)}; - const auto size{memory_manager.Read32(addr + 8)}; + const auto actual_addr{memory_manager.Read(addr)}; + const auto size{memory_manager.Read(addr + 8)}; // Look up global region in the cache based on address const auto& host_ptr{memory_manager.GetPointer(actual_addr)}; diff --git a/src/video_core/renderer_opengl/gl_rasterizer_cache.cpp b/src/video_core/renderer_opengl/gl_rasterizer_cache.cpp index 1133fa1f9..b94446428 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer_cache.cpp +++ b/src/video_core/renderer_opengl/gl_rasterizer_cache.cpp @@ -610,11 +610,11 @@ CachedSurface::CachedSurface(const SurfaceParams& params) // check is necessary to prevent flushing from overwriting unmapped memory. auto& memory_manager{Core::System::GetInstance().GPU().MemoryManager()}; - const u64 max_size{memory_manager.GetRegionEnd(params.gpu_addr) - params.gpu_addr}; - if (cached_size_in_bytes > max_size) { - LOG_ERROR(HW_GPU, "Surface size {} exceeds region size {}", params.size_in_bytes, max_size); - cached_size_in_bytes = max_size; - } + // const u64 max_size{memory_manager.GetRegionEnd(params.gpu_addr) - params.gpu_addr}; + // if (cached_size_in_bytes > max_size) { + // LOG_ERROR(HW_GPU, "Surface size {} exceeds region size {}", params.size_in_bytes, + // max_size); cached_size_in_bytes = max_size; + //} cpu_addr = *memory_manager.GpuToCpuAddress(params.gpu_addr); } From 21eb4cfa7f295205247397c117e1e2b4f6650d14 Mon Sep 17 00:00:00 2001 From: bunnei Date: Fri, 8 Mar 2019 21:10:12 -0500 Subject: [PATCH 3/9] gl_rasterizer_cache: Check that backing memory is valid before creating a surface. - Fixes a crash in Puyo Puyo Tetris. --- .../renderer_opengl/gl_rasterizer_cache.cpp | 22 ++++++------------- .../renderer_opengl/gl_rasterizer_cache.h | 5 +++++ 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/src/video_core/renderer_opengl/gl_rasterizer_cache.cpp b/src/video_core/renderer_opengl/gl_rasterizer_cache.cpp index b94446428..39dcf71ce 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer_cache.cpp +++ b/src/video_core/renderer_opengl/gl_rasterizer_cache.cpp @@ -564,6 +564,12 @@ void RasterizerCacheOpenGL::CopySurface(const Surface& src_surface, const Surfac CachedSurface::CachedSurface(const SurfaceParams& params) : params{params}, gl_target{SurfaceTargetToGL(params.target)}, cached_size_in_bytes{params.size_in_bytes}, RasterizerCacheObject{params.host_ptr} { + + const auto optional_cpu_addr{ + Core::System::GetInstance().GPU().MemoryManager().GpuToCpuAddress(params.gpu_addr)}; + ASSERT_MSG(optional_cpu_addr, "optional_cpu_addr is invalid"); + cpu_addr = *optional_cpu_addr; + texture.Create(gl_target); // TODO(Rodrigo): Using params.GetRect() returns a different size than using its Mip*(0) @@ -603,20 +609,6 @@ CachedSurface::CachedSurface(const SurfaceParams& params) ApplyTextureDefaults(texture.handle, params.max_mip_level); OpenGL::LabelGLObject(GL_TEXTURE, texture.handle, params.gpu_addr, params.IdentityString()); - - // Clamp size to mapped GPU memory region - // TODO(bunnei): Super Mario Odyssey maps a 0x40000 byte region and then uses it for a 0x80000 - // R32F render buffer. We do not yet know if this is a game bug or something else, but this - // check is necessary to prevent flushing from overwriting unmapped memory. - - auto& memory_manager{Core::System::GetInstance().GPU().MemoryManager()}; - // const u64 max_size{memory_manager.GetRegionEnd(params.gpu_addr) - params.gpu_addr}; - // if (cached_size_in_bytes > max_size) { - // LOG_ERROR(HW_GPU, "Surface size {} exceeds region size {}", params.size_in_bytes, - // max_size); cached_size_in_bytes = max_size; - //} - - cpu_addr = *memory_manager.GpuToCpuAddress(params.gpu_addr); } MICROPROFILE_DEFINE(OpenGL_SurfaceLoad, "OpenGL", "Surface Load", MP_RGB(128, 192, 64)); @@ -925,7 +917,7 @@ void RasterizerCacheOpenGL::LoadSurface(const Surface& surface) { } Surface RasterizerCacheOpenGL::GetSurface(const SurfaceParams& params, bool preserve_contents) { - if (params.gpu_addr == 0 || params.height * params.width == 0) { + if (!params.IsValid()) { return {}; } diff --git a/src/video_core/renderer_opengl/gl_rasterizer_cache.h b/src/video_core/renderer_opengl/gl_rasterizer_cache.h index d76bc0ee7..0efcafd07 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer_cache.h +++ b/src/video_core/renderer_opengl/gl_rasterizer_cache.h @@ -109,6 +109,11 @@ struct SurfaceParams { return size; } + /// Returns true if the parameters constitute a valid rasterizer surface. + bool IsValid() const { + return gpu_addr && host_ptr && height && width; + } + /// Returns the exact size of the memory occupied by a layer in a texture in VRAM, including /// mipmaps. std::size_t LayerMemorySize() const { From 197dcf0b5e426993f760374353cafb07126d45b2 Mon Sep 17 00:00:00 2001 From: bunnei Date: Sat, 9 Mar 2019 14:06:51 -0500 Subject: [PATCH 4/9] memory_manager: Add protections for invalid GPU addresses. - Avoid a crash in Xenoblade Chronicles 2. --- src/video_core/memory_manager.cpp | 54 +++++++++++++++++++++---------- src/video_core/memory_manager.h | 15 +++++---- 2 files changed, 45 insertions(+), 24 deletions(-) diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index 4c7faa067..e8edf9b14 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -90,32 +90,44 @@ GPUVAddr MemoryManager::FindFreeRegion(GPUVAddr region_start, u64 size, u64 alig return std::max(base, vma_handle->second.base); } -std::optional MemoryManager::GpuToCpuAddress(GPUVAddr gpu_addr) { - VAddr cpu_addr = page_table.backing_addr[gpu_addr >> page_bits]; +bool MemoryManager::IsAddressValid(GPUVAddr addr) const { + return (addr >> page_bits) < page_table.pointers.size(); +} + +std::optional MemoryManager::GpuToCpuAddress(GPUVAddr addr) { + if (!IsAddressValid(addr)) { + return {}; + } + + VAddr cpu_addr = page_table.backing_addr[addr >> page_bits]; if (cpu_addr) { - return cpu_addr + (gpu_addr & page_mask); + return cpu_addr + (addr & page_mask); } return {}; } template -T MemoryManager::Read(GPUVAddr vaddr) { - const u8* page_pointer = page_table.pointers[vaddr >> page_bits]; +T MemoryManager::Read(GPUVAddr addr) { + if (!IsAddressValid(addr)) { + return {}; + } + + const u8* page_pointer = page_table.pointers[addr >> page_bits]; if (page_pointer) { // NOTE: Avoid adding any extra logic to this fast-path block T value; - std::memcpy(&value, &page_pointer[vaddr & page_mask], sizeof(T)); + std::memcpy(&value, &page_pointer[addr & page_mask], sizeof(T)); return value; } - Common::PageType type = page_table.attributes[vaddr >> page_bits]; + Common::PageType type = page_table.attributes[addr >> page_bits]; switch (type) { case Common::PageType::Unmapped: - LOG_ERROR(HW_GPU, "Unmapped Read{} @ 0x{:08X}", sizeof(T) * 8, vaddr); + LOG_ERROR(HW_GPU, "Unmapped Read{} @ 0x{:08X}", sizeof(T) * 8, addr); return 0; case Common::PageType::Memory: - ASSERT_MSG(false, "Mapped memory page without a pointer @ {:016X}", vaddr); + ASSERT_MSG(false, "Mapped memory page without a pointer @ {:016X}", addr); break; default: UNREACHABLE(); @@ -124,22 +136,26 @@ T MemoryManager::Read(GPUVAddr vaddr) { } template -void MemoryManager::Write(GPUVAddr vaddr, T data) { - u8* page_pointer = page_table.pointers[vaddr >> page_bits]; - if (page_pointer) { - // NOTE: Avoid adding any extra logic to this fast-path block - std::memcpy(&page_pointer[vaddr & page_mask], &data, sizeof(T)); +void MemoryManager::Write(GPUVAddr addr, T data) { + if (!IsAddressValid(addr)) { return; } - Common::PageType type = page_table.attributes[vaddr >> page_bits]; + u8* page_pointer = page_table.pointers[addr >> page_bits]; + if (page_pointer) { + // NOTE: Avoid adding any extra logic to this fast-path block + std::memcpy(&page_pointer[addr & page_mask], &data, sizeof(T)); + return; + } + + Common::PageType type = page_table.attributes[addr >> page_bits]; switch (type) { case Common::PageType::Unmapped: LOG_ERROR(HW_GPU, "Unmapped Write{} 0x{:08X} @ 0x{:016X}", sizeof(data) * 8, - static_cast(data), vaddr); + static_cast(data), addr); return; case Common::PageType::Memory: - ASSERT_MSG(false, "Mapped memory page without a pointer @ {:016X}", vaddr); + ASSERT_MSG(false, "Mapped memory page without a pointer @ {:016X}", addr); break; default: UNREACHABLE(); @@ -156,6 +172,10 @@ template void MemoryManager::Write(GPUVAddr addr, u32 data); template void MemoryManager::Write(GPUVAddr addr, u64 data); u8* MemoryManager::GetPointer(GPUVAddr addr) { + if (!IsAddressValid(addr)) { + return {}; + } + u8* page_pointer = page_table.pointers[addr >> page_bits]; if (page_pointer) { return page_pointer + (addr & page_mask); diff --git a/src/video_core/memory_manager.h b/src/video_core/memory_manager.h index ac1b42936..76fa3d916 100644 --- a/src/video_core/memory_manager.h +++ b/src/video_core/memory_manager.h @@ -46,19 +46,19 @@ public: MemoryManager(); GPUVAddr AllocateSpace(u64 size, u64 align); - GPUVAddr AllocateSpace(GPUVAddr gpu_addr, u64 size, u64 align); + GPUVAddr AllocateSpace(GPUVAddr addr, u64 size, u64 align); GPUVAddr MapBufferEx(GPUVAddr cpu_addr, u64 size); - GPUVAddr MapBufferEx(GPUVAddr cpu_addr, GPUVAddr gpu_addr, u64 size); - GPUVAddr UnmapBuffer(GPUVAddr gpu_addr, u64 size); - std::optional GpuToCpuAddress(GPUVAddr gpu_addr); + GPUVAddr MapBufferEx(GPUVAddr cpu_addr, GPUVAddr addr, u64 size); + GPUVAddr UnmapBuffer(GPUVAddr addr, u64 size); + std::optional GpuToCpuAddress(GPUVAddr addr); template - T Read(GPUVAddr vaddr); + T Read(GPUVAddr addr); template - void Write(GPUVAddr vaddr, T data); + void Write(GPUVAddr addr, T data); - u8* GetPointer(GPUVAddr vaddr); + u8* GetPointer(GPUVAddr addr); void ReadBlock(GPUVAddr src_addr, void* dest_buffer, std::size_t size); void WriteBlock(GPUVAddr dest_addr, const void* src_buffer, std::size_t size); @@ -69,6 +69,7 @@ private: using VMAHandle = VMAMap::const_iterator; using VMAIter = VMAMap::iterator; + bool IsAddressValid(GPUVAddr addr) const; void MapPages(GPUVAddr base, u64 size, u8* memory, Common::PageType type, VAddr backing_addr = 0); void MapMemoryRegion(GPUVAddr base, u64 size, u8* target, VAddr backing_addr); From 19330f45d3d0efbf490a436c8689f30c4fc79e44 Mon Sep 17 00:00:00 2001 From: bunnei Date: Sat, 9 Mar 2019 14:36:52 -0500 Subject: [PATCH 5/9] maxwell_dma: Check for valid source in destination before copy. - Avoid a crash in Octopath Traveler. --- src/video_core/engines/maxwell_dma.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/video_core/engines/maxwell_dma.cpp b/src/video_core/engines/maxwell_dma.cpp index a0ded4c25..5cca5c29a 100644 --- a/src/video_core/engines/maxwell_dma.cpp +++ b/src/video_core/engines/maxwell_dma.cpp @@ -88,6 +88,16 @@ void MaxwellDMA::HandleCopy() { auto source_ptr{memory_manager.GetPointer(source)}; auto dst_ptr{memory_manager.GetPointer(dest)}; + if (!source_ptr) { + LOG_ERROR(HW_GPU, "source_ptr is invalid"); + return; + } + + if (!dst_ptr) { + LOG_ERROR(HW_GPU, "dst_ptr is invalid"); + return; + } + const auto FlushAndInvalidate = [&](u32 src_size, u64 dst_size) { // TODO(Subv): For now, manually flush the regions until we implement GPU-accelerated // copying. From 3ae0de9b53492dbcac7201a4ec7a3ada072fdfc0 Mon Sep 17 00:00:00 2001 From: bunnei Date: Mon, 18 Mar 2019 22:17:12 -0400 Subject: [PATCH 6/9] memory: Check that core is powered on before attempting to use GPU. - GPU will be released on shutdown, before pages are unmapped. - On subsequent runs, current_page_table will be not nullptr, but GPU might not be valid yet. --- src/core/memory.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/memory.cpp b/src/core/memory.cpp index 365ac82b4..332c1037c 100644 --- a/src/core/memory.cpp +++ b/src/core/memory.cpp @@ -48,7 +48,7 @@ static void MapPages(Common::PageTable& page_table, VAddr base, u64 size, u8* me (base + size) * PAGE_SIZE); // During boot, current_page_table might not be set yet, in which case we need not flush - if (current_page_table) { + if (Core::System::GetInstance().IsPoweredOn()) { Core::System::GetInstance().GPU().FlushAndInvalidateRegion(base << PAGE_BITS, size * PAGE_SIZE); } From 72837e4b3d312ac6d7e5114c7b6e370006d46921 Mon Sep 17 00:00:00 2001 From: bunnei Date: Wed, 20 Mar 2019 22:28:35 -0400 Subject: [PATCH 7/9] memory_manager: Bug fixes and further cleanup. --- src/video_core/memory_manager.cpp | 131 +++++++++++++++--------------- src/video_core/memory_manager.h | 14 ++-- 2 files changed, 72 insertions(+), 73 deletions(-) diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index e8edf9b14..0c4cf3974 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -40,7 +40,7 @@ GPUVAddr MemoryManager::AllocateSpace(GPUVAddr gpu_addr, u64 size, u64 align) { return gpu_addr; } -GPUVAddr MemoryManager::MapBufferEx(GPUVAddr cpu_addr, u64 size) { +GPUVAddr MemoryManager::MapBufferEx(VAddr cpu_addr, u64 size) { const GPUVAddr gpu_addr{ FindFreeRegion(address_space_base, size, page_size, VirtualMemoryArea::Type::Unmapped)}; MapBackingMemory(gpu_addr, Memory::GetPointer(cpu_addr), ((size + page_mask) & ~page_mask), @@ -48,7 +48,7 @@ GPUVAddr MemoryManager::MapBufferEx(GPUVAddr cpu_addr, u64 size) { return gpu_addr; } -GPUVAddr MemoryManager::MapBufferEx(GPUVAddr cpu_addr, GPUVAddr gpu_addr, u64 size) { +GPUVAddr MemoryManager::MapBufferEx(VAddr cpu_addr, GPUVAddr gpu_addr, u64 size) { ASSERT((gpu_addr & page_mask) == 0); MapBackingMemory(gpu_addr, Memory::GetPointer(cpu_addr), ((size + page_mask) & ~page_mask), @@ -74,20 +74,20 @@ GPUVAddr MemoryManager::FindFreeRegion(GPUVAddr region_start, u64 size, u64 alig align = (align + page_mask) & ~page_mask; // Find the first Free VMA. - const GPUVAddr base = region_start; - const VMAHandle vma_handle = std::find_if(vma_map.begin(), vma_map.end(), [&](const auto& vma) { - if (vma.second.type != vma_type) + const VMAHandle vma_handle{std::find_if(vma_map.begin(), vma_map.end(), [&](const auto& vma) { + if (vma.second.type != vma_type) { return false; + } - const VAddr vma_end = vma.second.base + vma.second.size; - return vma_end > base && vma_end >= base + size; - }); + const VAddr vma_end{vma.second.base + vma.second.size}; + return vma_end > region_start && vma_end >= region_start + size; + })}; if (vma_handle == vma_map.end()) { return {}; } - return std::max(base, vma_handle->second.base); + return std::max(region_start, vma_handle->second.base); } bool MemoryManager::IsAddressValid(GPUVAddr addr) const { @@ -99,7 +99,7 @@ std::optional MemoryManager::GpuToCpuAddress(GPUVAddr addr) { return {}; } - VAddr cpu_addr = page_table.backing_addr[addr >> page_bits]; + VAddr cpu_addr{page_table.backing_addr[addr >> page_bits]}; if (cpu_addr) { return cpu_addr + (addr & page_mask); } @@ -113,7 +113,7 @@ T MemoryManager::Read(GPUVAddr addr) { return {}; } - const u8* page_pointer = page_table.pointers[addr >> page_bits]; + const u8* page_pointer{page_table.pointers[addr >> page_bits]}; if (page_pointer) { // NOTE: Avoid adding any extra logic to this fast-path block T value; @@ -121,8 +121,7 @@ T MemoryManager::Read(GPUVAddr addr) { return value; } - Common::PageType type = page_table.attributes[addr >> page_bits]; - switch (type) { + switch (page_table.attributes[addr >> page_bits]) { case Common::PageType::Unmapped: LOG_ERROR(HW_GPU, "Unmapped Read{} @ 0x{:08X}", sizeof(T) * 8, addr); return 0; @@ -141,15 +140,14 @@ void MemoryManager::Write(GPUVAddr addr, T data) { return; } - u8* page_pointer = page_table.pointers[addr >> page_bits]; + u8* page_pointer{page_table.pointers[addr >> page_bits]}; if (page_pointer) { // NOTE: Avoid adding any extra logic to this fast-path block std::memcpy(&page_pointer[addr & page_mask], &data, sizeof(T)); return; } - Common::PageType type = page_table.attributes[addr >> page_bits]; - switch (type) { + switch (page_table.attributes[addr >> page_bits]) { case Common::PageType::Unmapped: LOG_ERROR(HW_GPU, "Unmapped Write{} 0x{:08X} @ 0x{:016X}", sizeof(data) * 8, static_cast(data), addr); @@ -176,7 +174,7 @@ u8* MemoryManager::GetPointer(GPUVAddr addr) { return {}; } - u8* page_pointer = page_table.pointers[addr >> page_bits]; + u8* page_pointer{page_table.pointers[addr >> page_bits]}; if (page_pointer) { return page_pointer + (addr & page_mask); } @@ -201,7 +199,7 @@ void MemoryManager::MapPages(GPUVAddr base, u64 size, u8* memory, Common::PageTy LOG_DEBUG(HW_GPU, "Mapping {} onto {:016X}-{:016X}", fmt::ptr(memory), base * page_size, (base + size) * page_size); - VAddr end = base + size; + const VAddr end{base + size}; ASSERT_MSG(end <= page_table.pointers.size(), "out of range mapping at {:016X}", base + page_table.pointers.size()); @@ -257,56 +255,58 @@ MemoryManager::VMAHandle MemoryManager::FindVMA(GPUVAddr target) const { } } +MemoryManager::VMAIter MemoryManager::Allocate(VMAIter vma_handle) { + VirtualMemoryArea& vma{vma_handle->second}; + + vma.type = VirtualMemoryArea::Type::Allocated; + vma.backing_addr = 0; + vma.backing_memory = {}; + UpdatePageTableForVMA(vma); + + return MergeAdjacent(vma_handle); +} + MemoryManager::VMAHandle MemoryManager::AllocateMemory(GPUVAddr target, std::size_t offset, u64 size) { // This is the appropriately sized VMA that will turn into our allocation. - VMAIter vma_handle = CarveVMA(target, size); - VirtualMemoryArea& final_vma = vma_handle->second; - ASSERT(final_vma.size == size); + VMAIter vma_handle{CarveVMA(target, size)}; + VirtualMemoryArea& vma{vma_handle->second}; - final_vma.type = VirtualMemoryArea::Type::Allocated; - final_vma.offset = offset; - UpdatePageTableForVMA(final_vma); + ASSERT(vma.size == size); - return MergeAdjacent(vma_handle); + vma.offset = offset; + + return Allocate(vma_handle); } MemoryManager::VMAHandle MemoryManager::MapBackingMemory(GPUVAddr target, u8* memory, u64 size, VAddr backing_addr) { // This is the appropriately sized VMA that will turn into our allocation. - VMAIter vma_handle = CarveVMA(target, size); - VirtualMemoryArea& final_vma = vma_handle->second; - ASSERT(final_vma.size == size); + VMAIter vma_handle{CarveVMA(target, size)}; + VirtualMemoryArea& vma{vma_handle->second}; - final_vma.type = VirtualMemoryArea::Type::Mapped; - final_vma.backing_memory = memory; - final_vma.backing_addr = backing_addr; - UpdatePageTableForVMA(final_vma); - - return MergeAdjacent(vma_handle); -} - -MemoryManager::VMAIter MemoryManager::Unmap(VMAIter vma_handle) { - VirtualMemoryArea& vma = vma_handle->second; - vma.type = VirtualMemoryArea::Type::Allocated; - vma.offset = 0; - vma.backing_memory = nullptr; + ASSERT(vma.size == size); + vma.type = VirtualMemoryArea::Type::Mapped; + vma.backing_memory = memory; + vma.backing_addr = backing_addr; UpdatePageTableForVMA(vma); return MergeAdjacent(vma_handle); } void MemoryManager::UnmapRange(GPUVAddr target, u64 size) { - VMAIter vma = CarveVMARange(target, size); - const VAddr target_end = target + size; + VMAIter vma{CarveVMARange(target, size)}; + const VAddr target_end{target + size}; + const VMAIter end{vma_map.end()}; - const VMAIter end = vma_map.end(); // The comparison against the end of the range must be done using addresses since VMAs can be // merged during this process, causing invalidation of the iterators. while (vma != end && vma->second.base < target_end) { - vma = std::next(Unmap(vma)); + // Unmapped ranges return to allocated state and can be reused + // This behavior is used by Super Mario Odyssey, Sonic Forces, and likely other games + vma = std::next(Allocate(vma)); } ASSERT(FindVMA(target)->second.size >= size); @@ -319,25 +319,26 @@ MemoryManager::VMAIter MemoryManager::StripIterConstness(const VMAHandle& iter) } MemoryManager::VMAIter MemoryManager::CarveVMA(GPUVAddr base, u64 size) { - ASSERT_MSG((size & Tegra::MemoryManager::page_mask) == 0, "non-page aligned size: 0x{:016X}", - size); - ASSERT_MSG((base & Tegra::MemoryManager::page_mask) == 0, "non-page aligned base: 0x{:016X}", - base); + ASSERT_MSG((size & page_mask) == 0, "non-page aligned size: 0x{:016X}", size); + ASSERT_MSG((base & page_mask) == 0, "non-page aligned base: 0x{:016X}", base); - VMAIter vma_handle = StripIterConstness(FindVMA(base)); + VMAIter vma_handle{StripIterConstness(FindVMA(base))}; if (vma_handle == vma_map.end()) { - // Target address is outside the range managed by the kernel + // Target address is outside the managed range return {}; } - const VirtualMemoryArea& vma = vma_handle->second; + const VirtualMemoryArea& vma{vma_handle->second}; if (vma.type == VirtualMemoryArea::Type::Mapped) { // Region is already allocated return {}; } - const VAddr start_in_vma = base - vma.base; - const VAddr end_in_vma = start_in_vma + size; + const VAddr start_in_vma{base - vma.base}; + const VAddr end_in_vma{start_in_vma + size}; + + ASSERT_MSG(end_in_vma <= vma.size, "region size 0x{:016X} is less than required size 0x{:016X}", + vma.size, end_in_vma); if (end_in_vma < vma.size) { // Split VMA at the end of the allocated region @@ -352,17 +353,15 @@ MemoryManager::VMAIter MemoryManager::CarveVMA(GPUVAddr base, u64 size) { } MemoryManager::VMAIter MemoryManager::CarveVMARange(GPUVAddr target, u64 size) { - ASSERT_MSG((size & Tegra::MemoryManager::page_mask) == 0, "non-page aligned size: 0x{:016X}", - size); - ASSERT_MSG((target & Tegra::MemoryManager::page_mask) == 0, "non-page aligned base: 0x{:016X}", - target); + ASSERT_MSG((size & page_mask) == 0, "non-page aligned size: 0x{:016X}", size); + ASSERT_MSG((target & page_mask) == 0, "non-page aligned base: 0x{:016X}", target); - const VAddr target_end = target + size; + const VAddr target_end{target + size}; ASSERT(target_end >= target); ASSERT(size > 0); - VMAIter begin_vma = StripIterConstness(FindVMA(target)); - const VMAIter i_end = vma_map.lower_bound(target_end); + VMAIter begin_vma{StripIterConstness(FindVMA(target))}; + const VMAIter i_end{vma_map.lower_bound(target_end)}; if (std::any_of(begin_vma, i_end, [](const auto& entry) { return entry.second.type == VirtualMemoryArea::Type::Unmapped; })) { @@ -373,7 +372,7 @@ MemoryManager::VMAIter MemoryManager::CarveVMARange(GPUVAddr target, u64 size) { begin_vma = SplitVMA(begin_vma, target - begin_vma->second.base); } - VMAIter end_vma = StripIterConstness(FindVMA(target_end)); + VMAIter end_vma{StripIterConstness(FindVMA(target_end))}; if (end_vma != vma_map.end() && target_end != end_vma->second.base) { end_vma = SplitVMA(end_vma, target_end - end_vma->second.base); } @@ -382,8 +381,8 @@ MemoryManager::VMAIter MemoryManager::CarveVMARange(GPUVAddr target, u64 size) { } MemoryManager::VMAIter MemoryManager::SplitVMA(VMAIter vma_handle, u64 offset_in_vma) { - VirtualMemoryArea& old_vma = vma_handle->second; - VirtualMemoryArea new_vma = old_vma; // Make a copy of the VMA + VirtualMemoryArea& old_vma{vma_handle->second}; + VirtualMemoryArea new_vma{old_vma}; // Make a copy of the VMA // For now, don't allow no-op VMA splits (trying to split at a boundary) because it's probably // a bug. This restriction might be removed later. @@ -411,14 +410,14 @@ MemoryManager::VMAIter MemoryManager::SplitVMA(VMAIter vma_handle, u64 offset_in } MemoryManager::VMAIter MemoryManager::MergeAdjacent(VMAIter iter) { - const VMAIter next_vma = std::next(iter); + const VMAIter next_vma{std::next(iter)}; if (next_vma != vma_map.end() && iter->second.CanBeMergedWith(next_vma->second)) { iter->second.size += next_vma->second.size; vma_map.erase(next_vma); } if (iter != vma_map.begin()) { - VMAIter prev_vma = std::prev(iter); + VMAIter prev_vma{std::prev(iter)}; if (prev_vma->second.CanBeMergedWith(iter->second)) { prev_vma->second.size += iter->second.size; vma_map.erase(iter); diff --git a/src/video_core/memory_manager.h b/src/video_core/memory_manager.h index 76fa3d916..60ba6b858 100644 --- a/src/video_core/memory_manager.h +++ b/src/video_core/memory_manager.h @@ -47,8 +47,8 @@ public: GPUVAddr AllocateSpace(u64 size, u64 align); GPUVAddr AllocateSpace(GPUVAddr addr, u64 size, u64 align); - GPUVAddr MapBufferEx(GPUVAddr cpu_addr, u64 size); - GPUVAddr MapBufferEx(GPUVAddr cpu_addr, GPUVAddr addr, u64 size); + GPUVAddr MapBufferEx(VAddr cpu_addr, u64 size); + GPUVAddr MapBufferEx(VAddr cpu_addr, GPUVAddr addr, u64 size); GPUVAddr UnmapBuffer(GPUVAddr addr, u64 size); std::optional GpuToCpuAddress(GPUVAddr addr); @@ -96,8 +96,8 @@ private: /// Converts a VMAHandle to a mutable VMAIter. VMAIter StripIterConstness(const VMAHandle& iter); - /// Unmaps the given VMA. - VMAIter Unmap(VMAIter vma); + /// Marks as the specfied VMA as allocated. + VMAIter Allocate(VMAIter vma); /** * Carves a VMA of a specific size at the specified address by splitting Free VMAs while doing @@ -135,11 +135,11 @@ private: static constexpr u64 page_mask{page_size - 1}; /// Address space in bits, this is fairly arbitrary but sufficiently large. - static constexpr u32 address_space_width = 39; + static constexpr u32 address_space_width{39}; /// Start address for mapping, this is fairly arbitrary but must be non-zero. - static constexpr GPUVAddr address_space_base = 0x100000; + static constexpr GPUVAddr address_space_base{0x100000}; /// End of address space, based on address space in bits. - static constexpr GPUVAddr address_space_end = 1ULL << address_space_width; + static constexpr GPUVAddr address_space_end{1ULL << address_space_width}; Common::PageTable page_table{page_bits}; VMAMap vma_map; From 5a5fccaa23f8670d85666efd6ea12b42883c4edc Mon Sep 17 00:00:00 2001 From: bunnei Date: Wed, 20 Mar 2019 22:58:49 -0400 Subject: [PATCH 8/9] memory_manager: Use Common::AlignUp in public interface as needed. --- src/video_core/memory_manager.cpp | 33 ++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index 0c4cf3974..6dc133c93 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -29,30 +29,39 @@ MemoryManager::MemoryManager() { } GPUVAddr MemoryManager::AllocateSpace(u64 size, u64 align) { + const u64 aligned_size{Common::AlignUp(size, page_size)}; const GPUVAddr gpu_addr{ - FindFreeRegion(address_space_base, size, align, VirtualMemoryArea::Type::Unmapped)}; - AllocateMemory(gpu_addr, 0, size); + FindFreeRegion(address_space_base, aligned_size, align, VirtualMemoryArea::Type::Unmapped)}; + + AllocateMemory(gpu_addr, 0, aligned_size); + return gpu_addr; } GPUVAddr MemoryManager::AllocateSpace(GPUVAddr gpu_addr, u64 size, u64 align) { - AllocateMemory(gpu_addr, 0, size); + const u64 aligned_size{Common::AlignUp(size, page_size)}; + + AllocateMemory(gpu_addr, 0, aligned_size); + return gpu_addr; } GPUVAddr MemoryManager::MapBufferEx(VAddr cpu_addr, u64 size) { - const GPUVAddr gpu_addr{ - FindFreeRegion(address_space_base, size, page_size, VirtualMemoryArea::Type::Unmapped)}; - MapBackingMemory(gpu_addr, Memory::GetPointer(cpu_addr), ((size + page_mask) & ~page_mask), - cpu_addr); + const u64 aligned_size{Common::AlignUp(size, page_size)}; + const GPUVAddr gpu_addr{FindFreeRegion(address_space_base, aligned_size, page_size, + VirtualMemoryArea::Type::Unmapped)}; + + MapBackingMemory(gpu_addr, Memory::GetPointer(cpu_addr), aligned_size, cpu_addr); + return gpu_addr; } GPUVAddr MemoryManager::MapBufferEx(VAddr cpu_addr, GPUVAddr gpu_addr, u64 size) { ASSERT((gpu_addr & page_mask) == 0); - MapBackingMemory(gpu_addr, Memory::GetPointer(cpu_addr), ((size + page_mask) & ~page_mask), - cpu_addr); + const u64 aligned_size{Common::AlignUp(size, page_size)}; + + MapBackingMemory(gpu_addr, Memory::GetPointer(cpu_addr), aligned_size, cpu_addr); return gpu_addr; } @@ -60,10 +69,12 @@ GPUVAddr MemoryManager::MapBufferEx(VAddr cpu_addr, GPUVAddr gpu_addr, u64 size) GPUVAddr MemoryManager::UnmapBuffer(GPUVAddr gpu_addr, u64 size) { ASSERT((gpu_addr & page_mask) == 0); + const u64 aligned_size{Common::AlignUp(size, page_size)}; const CacheAddr cache_addr{ToCacheAddr(GetPointer(gpu_addr))}; - Core::System::GetInstance().Renderer().Rasterizer().FlushAndInvalidateRegion(cache_addr, size); - UnmapRange(gpu_addr, ((size + page_mask) & ~page_mask)); + Core::System::GetInstance().Renderer().Rasterizer().FlushAndInvalidateRegion(cache_addr, + aligned_size); + UnmapRange(gpu_addr, aligned_size); return gpu_addr; } From 2117edd0f848cd7bc35bdbb1495ca10649715625 Mon Sep 17 00:00:00 2001 From: bunnei Date: Wed, 20 Mar 2019 23:12:28 -0400 Subject: [PATCH 9/9] memory_manager: Cleanup FindFreeRegion. --- src/video_core/memory_manager.cpp | 14 ++++---------- src/video_core/memory_manager.h | 4 ++-- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index 6dc133c93..e76b59842 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -30,8 +30,7 @@ MemoryManager::MemoryManager() { GPUVAddr MemoryManager::AllocateSpace(u64 size, u64 align) { const u64 aligned_size{Common::AlignUp(size, page_size)}; - const GPUVAddr gpu_addr{ - FindFreeRegion(address_space_base, aligned_size, align, VirtualMemoryArea::Type::Unmapped)}; + const GPUVAddr gpu_addr{FindFreeRegion(address_space_base, aligned_size)}; AllocateMemory(gpu_addr, 0, aligned_size); @@ -48,8 +47,7 @@ GPUVAddr MemoryManager::AllocateSpace(GPUVAddr gpu_addr, u64 size, u64 align) { GPUVAddr MemoryManager::MapBufferEx(VAddr cpu_addr, u64 size) { const u64 aligned_size{Common::AlignUp(size, page_size)}; - const GPUVAddr gpu_addr{FindFreeRegion(address_space_base, aligned_size, page_size, - VirtualMemoryArea::Type::Unmapped)}; + const GPUVAddr gpu_addr{FindFreeRegion(address_space_base, aligned_size)}; MapBackingMemory(gpu_addr, Memory::GetPointer(cpu_addr), aligned_size, cpu_addr); @@ -79,14 +77,10 @@ GPUVAddr MemoryManager::UnmapBuffer(GPUVAddr gpu_addr, u64 size) { return gpu_addr; } -GPUVAddr MemoryManager::FindFreeRegion(GPUVAddr region_start, u64 size, u64 align, - VirtualMemoryArea::Type vma_type) { - - align = (align + page_mask) & ~page_mask; - +GPUVAddr MemoryManager::FindFreeRegion(GPUVAddr region_start, u64 size) { // Find the first Free VMA. const VMAHandle vma_handle{std::find_if(vma_map.begin(), vma_map.end(), [&](const auto& vma) { - if (vma.second.type != vma_type) { + if (vma.second.type != VirtualMemoryArea::Type::Unmapped) { return false; } diff --git a/src/video_core/memory_manager.h b/src/video_core/memory_manager.h index 60ba6b858..34744bb27 100644 --- a/src/video_core/memory_manager.h +++ b/src/video_core/memory_manager.h @@ -126,8 +126,8 @@ private: /// Updates the pages corresponding to this VMA so they match the VMA's attributes. void UpdatePageTableForVMA(const VirtualMemoryArea& vma); - GPUVAddr FindFreeRegion(GPUVAddr region_start, u64 size, u64 align, - VirtualMemoryArea::Type vma_type); + /// Finds a free (unmapped region) of the specified size starting at the specified address. + GPUVAddr FindFreeRegion(GPUVAddr region_start, u64 size); private: static constexpr u64 page_bits{16};