From 049050856f30ba68e86197ac5cac622c3770e44b Mon Sep 17 00:00:00 2001 From: ReinUsesLisp Date: Sat, 29 Dec 2018 02:44:54 -0300 Subject: [PATCH 01/32] shader_decode: Implement LDG and basic cbuf tracking --- .../renderer_opengl/gl_shader_decompiler.h | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/video_core/renderer_opengl/gl_shader_decompiler.h b/src/video_core/renderer_opengl/gl_shader_decompiler.h index 0856a1361..a5bdbaf7a 100644 --- a/src/video_core/renderer_opengl/gl_shader_decompiler.h +++ b/src/video_core/renderer_opengl/gl_shader_decompiler.h @@ -92,6 +92,39 @@ private: std::string name; }; +class GlobalMemoryEntry { +public: + explicit GlobalMemoryEntry(u32 cbuf_index, u32 cbuf_offset, Maxwell::ShaderStage stage, + std::string name) + : cbuf_index{cbuf_index}, cbuf_offset{cbuf_offset}, stage{stage}, name{std::move(name)} {} + + u32 GetCbufIndex() const { + return cbuf_index; + } + + u32 GetCbufOffset() const { + return cbuf_offset; + } + + const std::string& GetName() const { + return name; + } + + Maxwell::ShaderStage GetStage() const { + return stage; + } + + u32 GetHash() const { + return (static_cast(stage) << 24) | (cbuf_index << 16) | cbuf_offset; + } + +private: + u32 cbuf_index{}; + u32 cbuf_offset{}; + Maxwell::ShaderStage stage{}; + std::string name; +}; + struct ShaderEntries { std::vector const_buffers; std::vector samplers; From 84412591c97e88b927c44b2517d70381608f1a24 Mon Sep 17 00:00:00 2001 From: ReinUsesLisp Date: Fri, 11 Jan 2019 01:46:49 -0300 Subject: [PATCH 02/32] file_util: Add shader directory --- src/common/common_paths.h | 1 + src/common/file_util.cpp | 1 + src/common/file_util.h | 1 + 3 files changed, 3 insertions(+) diff --git a/src/common/common_paths.h b/src/common/common_paths.h index 4f88de768..076752d3b 100644 --- a/src/common/common_paths.h +++ b/src/common/common_paths.h @@ -35,6 +35,7 @@ #define KEYS_DIR "keys" #define LOAD_DIR "load" #define DUMP_DIR "dump" +#define SHADER_DIR "shader" #define LOG_DIR "log" // Filenames diff --git a/src/common/file_util.cpp b/src/common/file_util.cpp index b52492da6..aecb66c32 100644 --- a/src/common/file_util.cpp +++ b/src/common/file_util.cpp @@ -710,6 +710,7 @@ const std::string& GetUserPath(UserPath path, const std::string& new_path) { paths.emplace(UserPath::NANDDir, user_path + NAND_DIR DIR_SEP); paths.emplace(UserPath::LoadDir, user_path + LOAD_DIR DIR_SEP); paths.emplace(UserPath::DumpDir, user_path + DUMP_DIR DIR_SEP); + paths.emplace(UserPath::ShaderDir, user_path + SHADER_DIR DIR_SEP); paths.emplace(UserPath::SysDataDir, user_path + SYSDATA_DIR DIR_SEP); paths.emplace(UserPath::KeysDir, user_path + KEYS_DIR DIR_SEP); // TODO: Put the logs in a better location for each OS diff --git a/src/common/file_util.h b/src/common/file_util.h index 571503d2a..38cc7f059 100644 --- a/src/common/file_util.h +++ b/src/common/file_util.h @@ -31,6 +31,7 @@ enum class UserPath { SDMCDir, LoadDir, DumpDir, + ShaderDir, SysDataDir, UserDir, }; From 0ed5d728ca12e407685b62802dec69b455f1a528 Mon Sep 17 00:00:00 2001 From: ReinUsesLisp Date: Sun, 13 Jan 2019 22:05:53 -0300 Subject: [PATCH 03/32] rasterizer_interface: Add disk cache entry for the rasterizer --- src/core/core.cpp | 3 +++ src/video_core/rasterizer_interface.h | 3 +++ src/video_core/renderer_opengl/gl_rasterizer.cpp | 5 +++++ src/video_core/renderer_opengl/gl_rasterizer.h | 1 + src/video_core/renderer_opengl/gl_shader_cache.cpp | 2 ++ src/video_core/renderer_opengl/gl_shader_cache.h | 3 +++ 6 files changed, 17 insertions(+) diff --git a/src/core/core.cpp b/src/core/core.cpp index 572814e4b..c8d7c442a 100644 --- a/src/core/core.cpp +++ b/src/core/core.cpp @@ -175,6 +175,9 @@ struct System::Impl { return static_cast(static_cast(ResultStatus::ErrorLoader) + static_cast(load_result)); } + + renderer->Rasterizer().LoadDiskResources(); + status = ResultStatus::Success; return status; } diff --git a/src/video_core/rasterizer_interface.h b/src/video_core/rasterizer_interface.h index 4c08bb148..bb4bc0e36 100644 --- a/src/video_core/rasterizer_interface.h +++ b/src/video_core/rasterizer_interface.h @@ -61,5 +61,8 @@ public: /// Increase/decrease the number of object in pages touching the specified region virtual void UpdatePagesCachedCount(Tegra::GPUVAddr addr, u64 size, int delta) {} + + /// Initialize disk cached resources for the game being emulated + virtual void LoadDiskResources() {} }; } // namespace VideoCore diff --git a/src/video_core/renderer_opengl/gl_rasterizer.cpp b/src/video_core/renderer_opengl/gl_rasterizer.cpp index c806b7da7..18aafe767 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer.cpp +++ b/src/video_core/renderer_opengl/gl_rasterizer.cpp @@ -22,6 +22,7 @@ #include "core/settings.h" #include "video_core/engines/maxwell_3d.h" #include "video_core/renderer_opengl/gl_rasterizer.h" +#include "video_core/renderer_opengl/gl_shader_cache.h" #include "video_core/renderer_opengl/gl_shader_gen.h" #include "video_core/renderer_opengl/maxwell_to_gl.h" #include "video_core/renderer_opengl/renderer_opengl.h" @@ -477,6 +478,10 @@ void RasterizerOpenGL::UpdatePagesCachedCount(VAddr addr, u64 size, int delta) { cached_pages.add({pages_interval, delta}); } +void RasterizerOpenGL::LoadDiskResources() { + shader_cache.LoadDiskCache(); +} + std::pair RasterizerOpenGL::ConfigureFramebuffers( OpenGLState& current_state, bool using_color_fb, bool using_depth_fb, bool preserve_contents, std::optional single_color_target) { diff --git a/src/video_core/renderer_opengl/gl_rasterizer.h b/src/video_core/renderer_opengl/gl_rasterizer.h index 7f2bf0f8b..ed7091f18 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer.h +++ b/src/video_core/renderer_opengl/gl_rasterizer.h @@ -60,6 +60,7 @@ public: u32 pixel_stride) override; bool AccelerateDrawBatch(bool is_indexed) override; void UpdatePagesCachedCount(Tegra::GPUVAddr addr, u64 size, int delta) override; + void LoadDiskResources() override; /// Maximum supported size that a constbuffer can have in bytes. static constexpr std::size_t MaxConstbufferSize = 0x10000; diff --git a/src/video_core/renderer_opengl/gl_shader_cache.cpp b/src/video_core/renderer_opengl/gl_shader_cache.cpp index 6174f7074..363b941f3 100644 --- a/src/video_core/renderer_opengl/gl_shader_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_cache.cpp @@ -225,6 +225,8 @@ void CachedShader::CalculateProperties() { ShaderCacheOpenGL::ShaderCacheOpenGL(RasterizerOpenGL& rasterizer) : RasterizerCache{rasterizer} {} +void ShaderCacheOpenGL::LoadDiskCache() {} + Shader ShaderCacheOpenGL::GetStageProgram(Maxwell::ShaderProgram program) { if (!Core::System::GetInstance().GPU().Maxwell3D().dirty_flags.shaders) { return last_shaders[static_cast(program)]; diff --git a/src/video_core/renderer_opengl/gl_shader_cache.h b/src/video_core/renderer_opengl/gl_shader_cache.h index 904d15dd0..5e72912f5 100644 --- a/src/video_core/renderer_opengl/gl_shader_cache.h +++ b/src/video_core/renderer_opengl/gl_shader_cache.h @@ -103,6 +103,9 @@ class ShaderCacheOpenGL final : public RasterizerCache { public: explicit ShaderCacheOpenGL(RasterizerOpenGL& rasterizer); + /// Loads disk cache for the current game + void LoadDiskCache(); + /// Gets the current specified shader stage program Shader GetStageProgram(Maxwell::ShaderProgram program); From 8b113686712d4418fdd2e5898ee25d8919b118db Mon Sep 17 00:00:00 2001 From: ReinUsesLisp Date: Sun, 13 Jan 2019 22:12:23 -0300 Subject: [PATCH 04/32] gl_shader_util: Add parameter to handle retrievable programs --- src/video_core/renderer_opengl/gl_resource_manager.cpp | 5 +++-- src/video_core/renderer_opengl/gl_resource_manager.h | 6 +++--- src/video_core/renderer_opengl/gl_shader_util.h | 5 ++++- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/video_core/renderer_opengl/gl_resource_manager.cpp b/src/video_core/renderer_opengl/gl_resource_manager.cpp index 4170cbd3c..bfe666a73 100644 --- a/src/video_core/renderer_opengl/gl_resource_manager.cpp +++ b/src/video_core/renderer_opengl/gl_resource_manager.cpp @@ -71,7 +71,8 @@ void OGLShader::Release() { } void OGLProgram::CreateFromSource(const char* vert_shader, const char* geo_shader, - const char* frag_shader, bool separable_program) { + const char* frag_shader, bool separable_program, + bool hint_retrievable) { OGLShader vert, geo, frag; if (vert_shader) vert.Create(vert_shader, GL_VERTEX_SHADER); @@ -81,7 +82,7 @@ void OGLProgram::CreateFromSource(const char* vert_shader, const char* geo_shade frag.Create(frag_shader, GL_FRAGMENT_SHADER); MICROPROFILE_SCOPE(OpenGL_ResourceCreation); - Create(separable_program, vert.handle, geo.handle, frag.handle); + Create(separable_program, hint_retrievable, vert.handle, geo.handle, frag.handle); } void OGLProgram::Release() { diff --git a/src/video_core/renderer_opengl/gl_resource_manager.h b/src/video_core/renderer_opengl/gl_resource_manager.h index df76cbc4b..fbb93ee49 100644 --- a/src/video_core/renderer_opengl/gl_resource_manager.h +++ b/src/video_core/renderer_opengl/gl_resource_manager.h @@ -101,15 +101,15 @@ public: } template - void Create(bool separable_program, T... shaders) { + void Create(bool separable_program, bool hint_retrievable, T... shaders) { if (handle != 0) return; - handle = GLShader::LoadProgram(separable_program, shaders...); + handle = GLShader::LoadProgram(separable_program, hint_retrievable, shaders...); } /// Creates a new internal OpenGL resource and stores the handle void CreateFromSource(const char* vert_shader, const char* geo_shader, const char* frag_shader, - bool separable_program = false); + bool separable_program = false, bool hint_retrievable = false); /// Deletes the internal OpenGL resource void Release(); diff --git a/src/video_core/renderer_opengl/gl_shader_util.h b/src/video_core/renderer_opengl/gl_shader_util.h index 285594f50..03b7548c2 100644 --- a/src/video_core/renderer_opengl/gl_shader_util.h +++ b/src/video_core/renderer_opengl/gl_shader_util.h @@ -47,7 +47,7 @@ GLuint LoadShader(const char* source, GLenum type); * @returns Handle of the newly created OpenGL program object */ template -GLuint LoadProgram(bool separable_program, T... shaders) { +GLuint LoadProgram(bool separable_program, bool hint_retrievable, T... shaders) { // Link the program LOG_DEBUG(Render_OpenGL, "Linking program..."); @@ -58,6 +58,9 @@ GLuint LoadProgram(bool separable_program, T... shaders) { if (separable_program) { glProgramParameteri(program_id, GL_PROGRAM_SEPARABLE, GL_TRUE); } + if (hint_retrievable) { + glProgramParameteri(program_id, GL_PROGRAM_BINARY_RETRIEVABLE_HINT, GL_TRUE); + } glLinkProgram(program_id); From c2c5260fd74bfd62723832ab690b222b464f16cd Mon Sep 17 00:00:00 2001 From: ReinUsesLisp Date: Sun, 13 Jan 2019 22:06:42 -0300 Subject: [PATCH 05/32] gl_shader_decompiler: Remove name entries --- .../renderer_opengl/gl_shader_decompiler.cpp | 8 ++--- .../renderer_opengl/gl_shader_decompiler.h | 30 +++++-------------- 2 files changed, 10 insertions(+), 28 deletions(-) diff --git a/src/video_core/renderer_opengl/gl_shader_decompiler.cpp b/src/video_core/renderer_opengl/gl_shader_decompiler.cpp index 36035d0d2..9184a1287 100644 --- a/src/video_core/renderer_opengl/gl_shader_decompiler.cpp +++ b/src/video_core/renderer_opengl/gl_shader_decompiler.cpp @@ -193,15 +193,13 @@ public: ShaderEntries GetShaderEntries() const { ShaderEntries entries; for (const auto& cbuf : ir.GetConstantBuffers()) { - entries.const_buffers.emplace_back(cbuf.second, stage, GetConstBufferBlock(cbuf.first), - cbuf.first); + entries.const_buffers.emplace_back(cbuf.second, stage, cbuf.first); } for (const auto& sampler : ir.GetSamplers()) { - entries.samplers.emplace_back(sampler, stage, GetSampler(sampler)); + entries.samplers.emplace_back(sampler, stage); } for (const auto& gmem : ir.GetGlobalMemoryBases()) { - entries.global_memory_entries.emplace_back(gmem.cbuf_index, gmem.cbuf_offset, stage, - GetGlobalMemoryBlock(gmem)); + entries.global_memory_entries.emplace_back(gmem.cbuf_index, gmem.cbuf_offset, stage); } entries.clip_distances = ir.GetClipDistances(); entries.shader_length = ir.GetLength(); diff --git a/src/video_core/renderer_opengl/gl_shader_decompiler.h b/src/video_core/renderer_opengl/gl_shader_decompiler.h index a5bdbaf7a..398be13d6 100644 --- a/src/video_core/renderer_opengl/gl_shader_decompiler.h +++ b/src/video_core/renderer_opengl/gl_shader_decompiler.h @@ -5,6 +5,7 @@ #pragma once #include +#include #include #include #include @@ -23,12 +24,8 @@ using Maxwell = Tegra::Engines::Maxwell3D::Regs; class ConstBufferEntry : public VideoCommon::Shader::ConstBuffer { public: explicit ConstBufferEntry(const VideoCommon::Shader::ConstBuffer& entry, - Maxwell::ShaderStage stage, const std::string& name, u32 index) - : VideoCommon::Shader::ConstBuffer{entry}, stage{stage}, name{name}, index{index} {} - - const std::string& GetName() const { - return name; - } + Maxwell::ShaderStage stage, u32 index) + : VideoCommon::Shader::ConstBuffer{entry}, stage{stage}, index{index} {} Maxwell::ShaderStage GetStage() const { return stage; @@ -39,35 +36,27 @@ public: } private: - std::string name; Maxwell::ShaderStage stage{}; u32 index{}; }; class SamplerEntry : public VideoCommon::Shader::Sampler { public: - explicit SamplerEntry(const VideoCommon::Shader::Sampler& entry, Maxwell::ShaderStage stage, - const std::string& name) - : VideoCommon::Shader::Sampler{entry}, stage{stage}, name{name} {} - - const std::string& GetName() const { - return name; - } + explicit SamplerEntry(const VideoCommon::Shader::Sampler& entry, Maxwell::ShaderStage stage) + : VideoCommon::Shader::Sampler{entry}, stage{stage} {} Maxwell::ShaderStage GetStage() const { return stage; } private: - std::string name; Maxwell::ShaderStage stage{}; }; class GlobalMemoryEntry { public: - explicit GlobalMemoryEntry(u32 cbuf_index, u32 cbuf_offset, Maxwell::ShaderStage stage, - std::string name) - : cbuf_index{cbuf_index}, cbuf_offset{cbuf_offset}, stage{stage}, name{std::move(name)} {} + explicit GlobalMemoryEntry(u32 cbuf_index, u32 cbuf_offset, Maxwell::ShaderStage stage) + : cbuf_index{cbuf_index}, cbuf_offset{cbuf_offset}, stage{stage} {} u32 GetCbufIndex() const { return cbuf_index; @@ -77,10 +66,6 @@ public: return cbuf_offset; } - const std::string& GetName() const { - return name; - } - Maxwell::ShaderStage GetStage() const { return stage; } @@ -122,7 +107,6 @@ private: u32 cbuf_index{}; u32 cbuf_offset{}; Maxwell::ShaderStage stage{}; - std::string name; }; struct ShaderEntries { From 145c3ac89e14c5400e617d4af08c3b8c251cf7c8 Mon Sep 17 00:00:00 2001 From: ReinUsesLisp Date: Sun, 13 Jan 2019 22:36:52 -0300 Subject: [PATCH 06/32] gl_shader_disk_cache: Add file and move BaseBindings declaration --- src/video_core/CMakeLists.txt | 2 + .../renderer_opengl/gl_shader_cache.h | 11 +---- .../renderer_opengl/gl_shader_disk_cache.cpp | 14 +++++++ .../renderer_opengl/gl_shader_disk_cache.h | 41 +++++++++++++++++++ 4 files changed, 58 insertions(+), 10 deletions(-) create mode 100644 src/video_core/renderer_opengl/gl_shader_disk_cache.cpp create mode 100644 src/video_core/renderer_opengl/gl_shader_disk_cache.h diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt index 6113e17ff..48be37082 100644 --- a/src/video_core/CMakeLists.txt +++ b/src/video_core/CMakeLists.txt @@ -44,6 +44,8 @@ add_library(video_core STATIC renderer_opengl/gl_shader_cache.h renderer_opengl/gl_shader_decompiler.cpp renderer_opengl/gl_shader_decompiler.h + renderer_opengl/gl_shader_disk_cache.cpp + renderer_opengl/gl_shader_disk_cache.h renderer_opengl/gl_shader_gen.cpp renderer_opengl/gl_shader_gen.h renderer_opengl/gl_shader_manager.cpp diff --git a/src/video_core/renderer_opengl/gl_shader_cache.h b/src/video_core/renderer_opengl/gl_shader_cache.h index 5e72912f5..18fb80bcc 100644 --- a/src/video_core/renderer_opengl/gl_shader_cache.h +++ b/src/video_core/renderer_opengl/gl_shader_cache.h @@ -16,6 +16,7 @@ #include "video_core/rasterizer_cache.h" #include "video_core/renderer_opengl/gl_resource_manager.h" #include "video_core/renderer_opengl/gl_shader_decompiler.h" +#include "video_core/renderer_opengl/gl_shader_disk_cache.h" #include "video_core/renderer_opengl/gl_shader_gen.h" namespace OpenGL { @@ -26,16 +27,6 @@ class RasterizerOpenGL; using Shader = std::shared_ptr; using Maxwell = Tegra::Engines::Maxwell3D::Regs; -struct BaseBindings { - u32 cbuf{}; - u32 gmem{}; - u32 sampler{}; - - bool operator<(const BaseBindings& rhs) const { - return std::tie(cbuf, gmem, sampler) < std::tie(rhs.cbuf, rhs.gmem, rhs.sampler); - } -}; - class CachedShader final : public RasterizerCacheObject { public: CachedShader(VAddr addr, Maxwell::ShaderProgram program_type); diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp new file mode 100644 index 000000000..b7876c3a7 --- /dev/null +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp @@ -0,0 +1,14 @@ +// Copyright 2019 yuzu Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#pragma once + +#include "video_core/renderer_opengl/gl_shader_disk_cache.h" + +namespace OpenGL { + +// Making sure sizes doesn't change by accident +static_assert(sizeof(BaseBindings) == 12); + +} // namespace OpenGL \ No newline at end of file diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.h b/src/video_core/renderer_opengl/gl_shader_disk_cache.h new file mode 100644 index 000000000..cb40e9926 --- /dev/null +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.h @@ -0,0 +1,41 @@ +// Copyright 2019 yuzu Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#pragma once + +#include + +#include "common/common_types.h" +#include "video_core/engines/maxwell_3d.h" + +namespace OpenGL { + +using ProgramCode = std::vector; +using Maxwell = Tegra::Engines::Maxwell3D::Regs; + +struct BaseBindings { +private: + auto Tie() const { + return std::tie(cbuf, gmem, sampler); + } + +public: + u32 cbuf{}; + u32 gmem{}; + u32 sampler{}; + + bool operator<(const BaseBindings& rhs) const { + return Tie() < rhs.Tie(); + } + + bool operator==(const BaseBindings& rhs) const { + return Tie() == rhs.Tie(); + } + + bool operator!=(const BaseBindings& rhs) const { + return !this->operator==(rhs); + } +}; + +} // namespace OpenGL \ No newline at end of file From 98be5a49289ca17b9470669ae94162dfa28eb2fb Mon Sep 17 00:00:00 2001 From: ReinUsesLisp Date: Sun, 13 Jan 2019 23:22:15 -0300 Subject: [PATCH 07/32] gl_shader_disk_cache: Add ShaderDiskCacheOpenGL class and helpers --- .../renderer_opengl/gl_shader_disk_cache.cpp | 52 +++++++++++++++++++ .../renderer_opengl/gl_shader_disk_cache.h | 24 +++++++++ 2 files changed, 76 insertions(+) diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp index b7876c3a7..de890676e 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp @@ -4,6 +4,18 @@ #pragma once +#include + +#include "common/assert.h" +#include "common/common_paths.h" +#include "common/common_types.h" +#include "common/file_util.h" +#include "common/logging/log.h" + +#include "core/core.h" +#include "core/hle/kernel/process.h" + +#include "video_core/renderer_opengl/gl_shader_cache.h" #include "video_core/renderer_opengl/gl_shader_disk_cache.h" namespace OpenGL { @@ -11,4 +23,44 @@ namespace OpenGL { // Making sure sizes doesn't change by accident static_assert(sizeof(BaseBindings) == 12); +namespace { +std::string GetTitleID() { + return fmt::format("{:016X}", Core::CurrentProcess()->GetTitleID()); +} +} // namespace + +bool ShaderDiskCacheOpenGL::EnsureDirectories() const { + const auto CreateDir = [](const std::string& dir) { + if (!FileUtil::CreateDir(dir)) { + LOG_ERROR(Render_OpenGL, "Failed to create directory={}", dir); + return false; + } + return true; + }; + + return CreateDir(FileUtil::GetUserPath(FileUtil::UserPath::ShaderDir)) && + CreateDir(GetBaseDir()) && CreateDir(GetTransferableDir()) && + CreateDir(GetPrecompiledDir()); +} + +std::string ShaderDiskCacheOpenGL::GetTransferablePath() const { + return FileUtil::SanitizePath(GetTransferableDir() + DIR_SEP_CHR + GetTitleID() + ".bin"); +} + +std::string ShaderDiskCacheOpenGL::GetPrecompiledPath() const { + return FileUtil::SanitizePath(GetPrecompiledDir() + DIR_SEP_CHR + GetTitleID() + ".bin"); +} + +std::string ShaderDiskCacheOpenGL::GetTransferableDir() const { + return GetBaseDir() + DIR_SEP "transferable"; +} + +std::string ShaderDiskCacheOpenGL::GetPrecompiledDir() const { + return GetBaseDir() + DIR_SEP "precompiled"; +} + +std::string ShaderDiskCacheOpenGL::GetBaseDir() const { + return FileUtil::GetUserPath(FileUtil::UserPath::ShaderDir) + DIR_SEP "opengl"; +} + } // namespace OpenGL \ No newline at end of file diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.h b/src/video_core/renderer_opengl/gl_shader_disk_cache.h index cb40e9926..690c92cae 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.h +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.h @@ -4,9 +4,11 @@ #pragma once +#include #include #include "common/common_types.h" +#include "common/file_util.h" #include "video_core/engines/maxwell_3d.h" namespace OpenGL { @@ -38,4 +40,26 @@ public: } }; +class ShaderDiskCacheOpenGL { +public: +private: + /// Create shader disk cache directories. Returns true on success. + bool EnsureDirectories() const; + + /// Gets current game's transferable file path + std::string GetTransferablePath() const; + + /// Gets current game's precompiled file path + std::string GetPrecompiledPath() const; + + /// Get user's transferable directory path + std::string GetTransferableDir() const; + + /// Get user's precompiled directory path + std::string GetPrecompiledDir() const; + + /// Get user's shader directory path + std::string GetBaseDir() const; +}; + } // namespace OpenGL \ No newline at end of file From b1efceec898b66ed1e940d887d376c5139272764 Mon Sep 17 00:00:00 2001 From: ReinUsesLisp Date: Sun, 13 Jan 2019 23:29:24 -0300 Subject: [PATCH 08/32] gl_shader_disk_cache: Add transferable stores --- .../renderer_opengl/gl_shader_disk_cache.cpp | 92 ++++++++++++++++ .../renderer_opengl/gl_shader_disk_cache.h | 102 ++++++++++++++++++ 2 files changed, 194 insertions(+) diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp index de890676e..ef8cfffd6 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp @@ -20,8 +20,16 @@ namespace OpenGL { +enum class EntryKind : u32 { + Raw, + Usage, +}; + +constexpr u32 NativeVersion = 1; + // Making sure sizes doesn't change by accident static_assert(sizeof(BaseBindings) == 12); +static_assert(sizeof(ShaderDiskCacheUsage) == 24); namespace { std::string GetTitleID() { @@ -29,6 +37,90 @@ std::string GetTitleID() { } } // namespace +ShaderDiskCacheRaw::ShaderDiskCacheRaw(FileUtil::IOFile& file) { + file.ReadBytes(&unique_identifier, sizeof(u64)); + file.ReadBytes(&program_type, sizeof(u32)); + + u32 program_code_size{}, program_code_size_b{}; + file.ReadBytes(&program_code_size, sizeof(u32)); + file.ReadBytes(&program_code_size_b, sizeof(u32)); + + program_code.resize(program_code_size); + program_code_b.resize(program_code_size_b); + + file.ReadArray(program_code.data(), program_code_size); + if (HasProgramA()) { + file.ReadArray(program_code_b.data(), program_code_size_b); + } +} + +void ShaderDiskCacheRaw::Save(FileUtil::IOFile& file) const { + file.WriteObject(unique_identifier); + file.WriteObject(static_cast(program_type)); + file.WriteObject(program_code_size); + file.WriteObject(program_code_size_b); + + file.WriteArray(program_code.data(), program_code_size); + if (HasProgramA()) { + file.WriteArray(program_code_b.data(), program_code_size_b); + } +} + +void ShaderDiskCacheOpenGL::SaveRaw(const ShaderDiskCacheRaw& entry) { + const u64 id = entry.GetUniqueIdentifier(); + if (transferable.find(id) != transferable.end()) { + // The shader already exists + return; + } + + FileUtil::IOFile file = AppendTransferableFile(); + if (!file.IsOpen()) { + return; + } + file.WriteObject(EntryKind::Raw); + entry.Save(file); + + transferable.insert({id, {}}); +} + +void ShaderDiskCacheOpenGL::SaveUsage(const ShaderDiskCacheUsage& usage) { + const auto it = transferable.find(usage.unique_identifier); + if (it == transferable.end()) { + LOG_CRITICAL(Render_OpenGL, "Saving shader usage without storing raw previously"); + UNREACHABLE(); + } + auto& usages{it->second}; + ASSERT(usages.find(usage) == usages.end()); + usages.insert(usage); + + FileUtil::IOFile file = AppendTransferableFile(); + if (!file.IsOpen()) { + return; + } + file.WriteObject(EntryKind::Usage); + file.WriteObject(usage); +} + +FileUtil::IOFile ShaderDiskCacheOpenGL::AppendTransferableFile() const { + if (!EnsureDirectories()) { + return {}; + } + + const auto transferable_path{GetTransferablePath()}; + const bool existed = FileUtil::Exists(transferable_path); + + FileUtil::IOFile file(transferable_path, "ab"); + if (!file.IsOpen()) { + LOG_ERROR(Render_OpenGL, "Failed to open transferable cache in path={}", transferable_path); + return {}; + } + if (!existed || file.GetSize() == 0) { + // If the file didn't exist, write its version + file.WriteObject(NativeVersion); + } + return file; +} + bool ShaderDiskCacheOpenGL::EnsureDirectories() const { const auto CreateDir = [](const std::string& dir) { if (!FileUtil::CreateDir(dir)) { diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.h b/src/video_core/renderer_opengl/gl_shader_disk_cache.h index 690c92cae..d4449c132 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.h +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.h @@ -4,12 +4,18 @@ #pragma once +#include #include #include +#include +#include + +#include "common/assert.h" #include "common/common_types.h" #include "common/file_util.h" #include "video_core/engines/maxwell_3d.h" +#include "video_core/renderer_opengl/gl_shader_gen.h" namespace OpenGL { @@ -40,9 +46,102 @@ public: } }; +class ShaderDiskCacheRaw { +public: + explicit ShaderDiskCacheRaw(FileUtil::IOFile& file); + + explicit ShaderDiskCacheRaw(u64 unique_identifier, Maxwell::ShaderProgram program_type, + u32 program_code_size, u32 program_code_size_b, + ProgramCode program_code, ProgramCode program_code_b) + : unique_identifier{unique_identifier}, program_type{program_type}, + program_code_size{program_code_size}, program_code_size_b{program_code_size_b}, + program_code{std::move(program_code)}, program_code_b{std::move(program_code_b)} {} + + void Save(FileUtil::IOFile& file) const; + + u64 GetUniqueIdentifier() const { + return unique_identifier; + } + + bool HasProgramA() const { + return program_type == Maxwell::ShaderProgram::VertexA; + } + + Maxwell::ShaderProgram GetProgramType() const { + return program_type; + } + + Maxwell::ShaderStage GetProgramStage() const { + switch (program_type) { + case Maxwell::ShaderProgram::VertexA: + case Maxwell::ShaderProgram::VertexB: + return Maxwell::ShaderStage::Vertex; + case Maxwell::ShaderProgram::TesselationControl: + return Maxwell::ShaderStage::TesselationControl; + case Maxwell::ShaderProgram::TesselationEval: + return Maxwell::ShaderStage::TesselationEval; + case Maxwell::ShaderProgram::Geometry: + return Maxwell::ShaderStage::Geometry; + case Maxwell::ShaderProgram::Fragment: + return Maxwell::ShaderStage::Fragment; + } + UNREACHABLE(); + } + + const ProgramCode& GetProgramCode() const { + return program_code; + } + + const ProgramCode& GetProgramCodeB() const { + return program_code_b; + } + +private: + u64 unique_identifier{}; + Maxwell::ShaderProgram program_type{}; + u32 program_code_size{}; + u32 program_code_size_b{}; + + ProgramCode program_code; + ProgramCode program_code_b; +}; + +struct ShaderDiskCacheUsage { +private: + auto Tie() const { + return std::tie(unique_identifier, bindings, primitive); + } + +public: + u64 unique_identifier{}; + BaseBindings bindings; + GLenum primitive{}; + + bool operator<(const ShaderDiskCacheUsage& rhs) const { + return Tie() < rhs.Tie(); + } + + bool operator==(const ShaderDiskCacheUsage& rhs) const { + return Tie() == rhs.Tie(); + } + + bool operator!=(const ShaderDiskCacheUsage& rhs) const { + return !this->operator==(rhs); + } +}; + class ShaderDiskCacheOpenGL { public: + /// Saves a raw dump to the transferable file. Checks for collisions. + void SaveRaw(const ShaderDiskCacheRaw& entry); + + /// Saves shader usage to the transferable file. Does not check for collisions. + void SaveUsage(const ShaderDiskCacheUsage& usage); + private: + /// Opens current game's transferable file and write it's header if it doesn't exist + FileUtil::IOFile AppendTransferableFile() const; + /// Create shader disk cache directories. Returns true on success. bool EnsureDirectories() const; @@ -60,6 +159,9 @@ private: /// Get user's shader directory path std::string GetBaseDir() const; + + // Stored transferable shaders + std::map> transferable; }; } // namespace OpenGL \ No newline at end of file From 3435cd8d5e34f36ae493840fb5577b055a6b9fe2 Mon Sep 17 00:00:00 2001 From: ReinUsesLisp Date: Sun, 13 Jan 2019 23:30:37 -0300 Subject: [PATCH 09/32] gl_shader_disk_cache: Add transferable load --- .../renderer_opengl/gl_shader_disk_cache.cpp | 52 +++++++++++++++++++ .../renderer_opengl/gl_shader_disk_cache.h | 4 ++ 2 files changed, 56 insertions(+) diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp index ef8cfffd6..eb9854b9f 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp @@ -66,6 +66,58 @@ void ShaderDiskCacheRaw::Save(FileUtil::IOFile& file) const { } } +bool ShaderDiskCacheOpenGL::LoadTransferable(std::vector& raws, + std::vector& usages) { + FileUtil::IOFile file(GetTransferablePath(), "rb"); + if (!file.IsOpen()) { + LOG_INFO(Render_OpenGL, "No transferable shader cache found for game with title id={}", + GetTitleID()); + return false; + } + const u64 file_size = file.GetSize(); + + u32 version{}; + file.ReadBytes(&version, sizeof(version)); + + if (version < NativeVersion) { + LOG_INFO(Render_OpenGL, "Transferable shader cache is old - removing"); + file.Close(); + FileUtil::Delete(GetTransferablePath()); + return false; + } + if (version > NativeVersion) { + LOG_WARNING(Render_OpenGL, "Transferable shader cache was generated with a newer version " + "of the emulator - skipping"); + return false; + } + + // Version is valid, load the shaders + while (file.Tell() < file_size) { + EntryKind kind{}; + file.ReadBytes(&kind, sizeof(u32)); + + switch (kind) { + case EntryKind::Raw: { + ShaderDiskCacheRaw entry{file}; + transferable.insert({entry.GetUniqueIdentifier(), {}}); + raws.push_back(std::move(entry)); + break; + } + case EntryKind::Usage: { + ShaderDiskCacheUsage usage{}; + file.ReadBytes(&usage, sizeof(usage)); + usages.push_back(std::move(usage)); + break; + } + default: + LOG_ERROR(Render_OpenGL, "Unknown transferable shader cache entry kind={} - aborting", + static_cast(kind)); + return false; + } + } + return true; +} + void ShaderDiskCacheOpenGL::SaveRaw(const ShaderDiskCacheRaw& entry) { const u64 id = entry.GetUniqueIdentifier(); if (transferable.find(id) != transferable.end()) { diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.h b/src/video_core/renderer_opengl/gl_shader_disk_cache.h index d4449c132..46d762b64 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.h +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.h @@ -132,6 +132,10 @@ public: class ShaderDiskCacheOpenGL { public: + /// Loads transferable cache. If file has a old version, it deletes it. Returns true on success. + bool LoadTransferable(std::vector& raws, + std::vector& usages); + /// Saves a raw dump to the transferable file. Checks for collisions. void SaveRaw(const ShaderDiskCacheRaw& entry); From 57fb15d2a3fcc6512145c5a46c18f168994a65b0 Mon Sep 17 00:00:00 2001 From: ReinUsesLisp Date: Sun, 13 Jan 2019 23:33:00 -0300 Subject: [PATCH 10/32] gl_shader_disk_cache: Add precompiled save --- .../renderer_opengl/gl_shader_disk_cache.cpp | 43 +++++++++++++++++++ .../renderer_opengl/gl_shader_disk_cache.h | 14 ++++++ 2 files changed, 57 insertions(+) diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp index eb9854b9f..0c42e3d8a 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp @@ -27,6 +27,9 @@ enum class EntryKind : u32 { constexpr u32 NativeVersion = 1; +// TODO(Rodrigo): Hash files +constexpr u64 PrecompiledHash = 0xdeadbeefdeadbeef; + // Making sure sizes doesn't change by accident static_assert(sizeof(BaseBindings) == 12); static_assert(sizeof(ShaderDiskCacheUsage) == 24); @@ -153,6 +156,26 @@ void ShaderDiskCacheOpenGL::SaveUsage(const ShaderDiskCacheUsage& usage) { file.WriteObject(usage); } +void ShaderDiskCacheOpenGL::SavePrecompiled(const ShaderDiskCacheUsage& usage, GLuint program) { + FileUtil::IOFile file = AppendPrecompiledFile(); + if (!file.IsOpen()) { + return; + } + + file.WriteObject(usage); + + GLint binary_length{}; + glGetProgramiv(program, GL_PROGRAM_BINARY_LENGTH, &binary_length); + + GLenum binary_format{}; + std::vector binary(binary_length); + glGetProgramBinary(program, binary_length, nullptr, &binary_format, binary.data()); + + file.WriteObject(static_cast(binary_format)); + file.WriteObject(static_cast(binary_length)); + file.WriteArray(binary.data(), binary.size()); +} + FileUtil::IOFile ShaderDiskCacheOpenGL::AppendTransferableFile() const { if (!EnsureDirectories()) { return {}; @@ -173,6 +196,26 @@ FileUtil::IOFile ShaderDiskCacheOpenGL::AppendTransferableFile() const { return file; } +FileUtil::IOFile ShaderDiskCacheOpenGL::AppendPrecompiledFile() const { + if (!EnsureDirectories()) { + return {}; + } + + const auto precompiled_path{GetPrecompiledPath()}; + const bool existed = FileUtil::Exists(precompiled_path); + + FileUtil::IOFile file(precompiled_path, "ab"); + if (!file.IsOpen()) { + LOG_ERROR(Render_OpenGL, "Failed to open precompiled cache in path={}", precompiled_path); + return {}; + } + + if (!existed || file.GetSize() == 0) { + file.WriteObject(PrecompiledHash); + } + return file; +} + bool ShaderDiskCacheOpenGL::EnsureDirectories() const { const auto CreateDir = [](const std::string& dir) { if (!FileUtil::CreateDir(dir)) { diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.h b/src/video_core/renderer_opengl/gl_shader_disk_cache.h index 46d762b64..fdb29caa5 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.h +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.h @@ -130,6 +130,14 @@ public: } }; +struct ShaderDiskCachePrecompiledEntry { + ShaderDiskCacheUsage usage; + GLenum binary_format; + std::vector binary; + std::string code; + GLShader::ShaderEntries entries; +}; + class ShaderDiskCacheOpenGL { public: /// Loads transferable cache. If file has a old version, it deletes it. Returns true on success. @@ -142,10 +150,16 @@ public: /// Saves shader usage to the transferable file. Does not check for collisions. void SaveUsage(const ShaderDiskCacheUsage& usage); + /// Saves a precompiled shader entry. Does not check for collisions. + void SavePrecompiled(const ShaderDiskCacheUsage& usage, GLuint program); + private: /// Opens current game's transferable file and write it's header if it doesn't exist FileUtil::IOFile AppendTransferableFile() const; + /// Opens current game's precompiled file and write it's header if it doesn't exist + FileUtil::IOFile AppendPrecompiledFile() const; + /// Create shader disk cache directories. Returns true on success. bool EnsureDirectories() const; From a1faed99501d04b233d44a7cab0368449d8a9cb5 Mon Sep 17 00:00:00 2001 From: ReinUsesLisp Date: Sun, 13 Jan 2019 23:33:31 -0300 Subject: [PATCH 11/32] gl_shader_disk_cache: Add precompiled load --- .../renderer_opengl/gl_shader_disk_cache.cpp | 39 +++++++++++++++++++ .../renderer_opengl/gl_shader_disk_cache.h | 6 +++ 2 files changed, 45 insertions(+) diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp index 0c42e3d8a..a87bb1f9e 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp @@ -121,6 +121,45 @@ bool ShaderDiskCacheOpenGL::LoadTransferable(std::vector& ra return true; } +std::vector ShaderDiskCacheOpenGL::LoadPrecompiled() { + FileUtil::IOFile file(GetPrecompiledPath(), "rb"); + if (!file.IsOpen()) { + LOG_INFO(Render_OpenGL, "No precompiled shader cache found for game with title id={}", + GetTitleID()); + return {}; + } + const u64 file_size = file.GetSize(); + + u64 precompiled_hash{}; + file.ReadBytes(&precompiled_hash, sizeof(precompiled_hash)); + if (precompiled_hash != PrecompiledHash) { + LOG_INFO(Render_OpenGL, "Precompiled cache is from another version of yuzu - removing"); + file.Close(); + InvalidatePrecompiled(); + return {}; + } + + std::vector precompiled; + while (file.Tell() < file_size) { + ShaderDiskCachePrecompiledEntry entry; + file.ReadBytes(&entry.usage, sizeof(entry.usage)); + + file.ReadBytes(&entry.binary_format, sizeof(u32)); + + u32 binary_length{}; + file.ReadBytes(&binary_length, sizeof(u32)); + entry.binary.resize(binary_length); + file.ReadBytes(entry.binary.data(), entry.binary.size()); + + precompiled.push_back(entry); + } + return precompiled; +} + +void ShaderDiskCacheOpenGL::InvalidatePrecompiled() const { + FileUtil::Delete(GetPrecompiledPath()); +} + void ShaderDiskCacheOpenGL::SaveRaw(const ShaderDiskCacheRaw& entry) { const u64 id = entry.GetUniqueIdentifier(); if (transferable.find(id) != transferable.end()) { diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.h b/src/video_core/renderer_opengl/gl_shader_disk_cache.h index fdb29caa5..e68aea694 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.h +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.h @@ -144,6 +144,12 @@ public: bool LoadTransferable(std::vector& raws, std::vector& usages); + /// Loads current game's precompiled cache. Invalidates if emulator's version has changed. + std::vector LoadPrecompiled(); + + /// Removes the precompiled cache file. + void InvalidatePrecompiled() const; + /// Saves a raw dump to the transferable file. Checks for collisions. void SaveRaw(const ShaderDiskCacheRaw& entry); From 403908622670d11531e132e5bbab13a5f4cf05ce Mon Sep 17 00:00:00 2001 From: ReinUsesLisp Date: Mon, 14 Jan 2019 01:25:25 -0300 Subject: [PATCH 12/32] gl_shader_disk_cache: Add transferable cache invalidation --- src/video_core/renderer_opengl/gl_shader_disk_cache.cpp | 5 +++++ src/video_core/renderer_opengl/gl_shader_disk_cache.h | 3 +++ 2 files changed, 8 insertions(+) diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp index a87bb1f9e..4d02a800d 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp @@ -156,6 +156,11 @@ std::vector ShaderDiskCacheOpenGL::LoadPrecompi return precompiled; } +void ShaderDiskCacheOpenGL::InvalidateTransferable() const { + FileUtil::Delete(GetTransferablePath()); + InvalidatePrecompiled(); +} + void ShaderDiskCacheOpenGL::InvalidatePrecompiled() const { FileUtil::Delete(GetPrecompiledPath()); } diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.h b/src/video_core/renderer_opengl/gl_shader_disk_cache.h index e68aea694..6c4c7bd5c 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.h +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.h @@ -147,6 +147,9 @@ public: /// Loads current game's precompiled cache. Invalidates if emulator's version has changed. std::vector LoadPrecompiled(); + /// Removes the transferable (and precompiled) cache file. + void InvalidateTransferable() const; + /// Removes the precompiled cache file. void InvalidatePrecompiled() const; From a3703f5767332dfc5f7e8d37a1f715d8ccb76fcf Mon Sep 17 00:00:00 2001 From: ReinUsesLisp Date: Mon, 14 Jan 2019 00:58:15 -0300 Subject: [PATCH 13/32] gl_shader_cache: Refactor to support disk shader cache --- .../renderer_opengl/gl_shader_cache.cpp | 458 +++++++++++++----- .../renderer_opengl/gl_shader_cache.h | 59 ++- 2 files changed, 392 insertions(+), 125 deletions(-) diff --git a/src/video_core/renderer_opengl/gl_shader_cache.cpp b/src/video_core/renderer_opengl/gl_shader_cache.cpp index 363b941f3..a70ff79d0 100644 --- a/src/video_core/renderer_opengl/gl_shader_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_cache.cpp @@ -11,6 +11,7 @@ #include "video_core/renderer_opengl/gl_rasterizer.h" #include "video_core/renderer_opengl/gl_shader_cache.h" #include "video_core/renderer_opengl/gl_shader_decompiler.h" +#include "video_core/renderer_opengl/gl_shader_disk_cache.h" #include "video_core/renderer_opengl/gl_shader_manager.h" #include "video_core/renderer_opengl/utils.h" #include "video_core/shader/shader_ir.h" @@ -19,8 +20,19 @@ namespace OpenGL { using VideoCommon::Shader::ProgramCode; +// One UBO is always reserved for emulation values +constexpr u32 RESERVED_UBOS = 1; + +struct UnspecializedShader { + std::string code; + GLShader::ShaderEntries entries; + Maxwell::ShaderProgram program_type; +}; + +namespace { + /// Gets the address for the specified shader stage program -static VAddr GetShaderAddress(Maxwell::ShaderProgram program) { +VAddr GetShaderAddress(Maxwell::ShaderProgram program) { const auto& gpu = Core::System::GetInstance().GPU().Maxwell3D(); const auto& shader_config = gpu.regs.shader_config[static_cast(program)]; const auto address = gpu.memory_manager.GpuToCpuAddress(gpu.regs.code_address.CodeAddress() + @@ -30,7 +42,7 @@ static VAddr GetShaderAddress(Maxwell::ShaderProgram program) { } /// Gets the shader program code from memory for the specified address -static ProgramCode GetShaderCode(VAddr addr) { +ProgramCode GetShaderCode(VAddr addr) { ProgramCode program_code(VideoCommon::Shader::MAX_PROGRAM_LENGTH); Memory::ReadBlock(addr, program_code.data(), program_code.size() * sizeof(u64)); return program_code; @@ -51,38 +63,193 @@ constexpr GLenum GetShaderType(Maxwell::ShaderProgram program_type) { } } -CachedShader::CachedShader(VAddr addr, Maxwell::ShaderProgram program_type) - : addr{addr}, program_type{program_type}, setup{GetShaderCode(addr)} { +/// Gets if the current instruction offset is a scheduler instruction +constexpr bool IsSchedInstruction(std::size_t offset, std::size_t main_offset) { + // Sched instructions appear once every 4 instructions. + constexpr std::size_t SchedPeriod = 4; + const std::size_t absolute_offset = offset - main_offset; + return (absolute_offset % SchedPeriod) == 0; +} - GLShader::ProgramResult program_result; +/// Describes primitive behavior on geometry shaders +constexpr std::tuple GetPrimitiveDescription(GLenum primitive_mode) { + switch (primitive_mode) { + case GL_POINTS: + return {"points", "Points", 1}; + case GL_LINES: + case GL_LINE_STRIP: + return {"lines", "Lines", 2}; + case GL_LINES_ADJACENCY: + case GL_LINE_STRIP_ADJACENCY: + return {"lines_adjacency", "LinesAdj", 4}; + case GL_TRIANGLES: + case GL_TRIANGLE_STRIP: + case GL_TRIANGLE_FAN: + return {"triangles", "Triangles", 3}; + case GL_TRIANGLES_ADJACENCY: + case GL_TRIANGLE_STRIP_ADJACENCY: + return {"triangles_adjacency", "TrianglesAdj", 6}; + default: + return {"points", "Invalid", 1}; + } +} - switch (program_type) { - case Maxwell::ShaderProgram::VertexA: +/// Calculates the size of a program stream +std::size_t CalculateProgramSize(const GLShader::ProgramCode& program) { + constexpr std::size_t start_offset = 10; + std::size_t offset = start_offset; + std::size_t size = start_offset * sizeof(u64); + while (offset < program.size()) { + const u64 instruction = program[offset]; + if (!IsSchedInstruction(offset, start_offset)) { + if (instruction == 0 || (instruction >> 52) == 0x50b) { + // End on Maxwell's "nop" instruction + break; + } + } + size += sizeof(u64); + offset++; + } + // The last instruction is included in the program size + return std::min(size + sizeof(u64), program.size() * sizeof(u64)); +} + +/// Hashes one (or two) program streams +u64 GetUniqueIdentifier(Maxwell::ShaderProgram program_type, const ProgramCode& code, + const ProgramCode& code_b) { + u64 unique_identifier = + Common::CityHash64(reinterpret_cast(code.data()), CalculateProgramSize(code)); + if (program_type != Maxwell::ShaderProgram::VertexA) { + return unique_identifier; + } + // VertexA programs include two programs + + std::size_t seed = 0; + boost::hash_combine(seed, unique_identifier); + + const u64 identifier_b = Common::CityHash64(reinterpret_cast(code_b.data()), + CalculateProgramSize(code_b)); + boost::hash_combine(seed, identifier_b); + return static_cast(seed); +} + +/// Creates an unspecialized program from code streams +GLShader::ProgramResult CreateProgram(Maxwell::ShaderProgram program_type, ProgramCode program_code, + ProgramCode program_code_b) { + GLShader::ShaderSetup setup(std::move(program_code)); + if (program_type == Maxwell::ShaderProgram::VertexA) { // VertexB is always enabled, so when VertexA is enabled, we have two vertex shaders. // Conventional HW does not support this, so we combine VertexA and VertexB into one // stage here. - setup.SetProgramB(GetShaderCode(GetShaderAddress(Maxwell::ShaderProgram::VertexB))); + setup.SetProgramB(std::move(program_code_b)); + } + + switch (program_type) { + case Maxwell::ShaderProgram::VertexA: case Maxwell::ShaderProgram::VertexB: - CalculateProperties(); - program_result = GLShader::GenerateVertexShader(setup); - break; + return GLShader::GenerateVertexShader(setup); case Maxwell::ShaderProgram::Geometry: - CalculateProperties(); - program_result = GLShader::GenerateGeometryShader(setup); - break; + return GLShader::GenerateGeometryShader(setup); case Maxwell::ShaderProgram::Fragment: - CalculateProperties(); - program_result = GLShader::GenerateFragmentShader(setup); - break; + return GLShader::GenerateFragmentShader(setup); default: LOG_CRITICAL(HW_GPU, "Unimplemented program_type={}", static_cast(program_type)); UNREACHABLE(); + return {}; + } +} + +CachedProgram SpecializeShader(const std::string& code, const GLShader::ShaderEntries& entries, + Maxwell::ShaderProgram program_type, BaseBindings base_bindings, + GLenum primitive_mode, bool hint_retrievable = false) { + std::string source = "#version 430 core\n"; + source += fmt::format("#define EMULATION_UBO_BINDING {}\n", base_bindings.cbuf++); + + for (const auto& cbuf : entries.const_buffers) { + source += + fmt::format("#define CBUF_BINDING_{} {}\n", cbuf.GetIndex(), base_bindings.cbuf++); + } + for (const auto& gmem : entries.global_memory_entries) { + source += fmt::format("#define GMEM_BINDING_{}_{} {}\n", gmem.GetCbufIndex(), + gmem.GetCbufOffset(), base_bindings.gmem++); + } + for (const auto& sampler : entries.samplers) { + source += fmt::format("#define SAMPLER_BINDING_{} {}\n", sampler.GetIndex(), + base_bindings.sampler++); + } + + if (program_type == Maxwell::ShaderProgram::Geometry) { + const auto [glsl_topology, _, max_vertices] = GetPrimitiveDescription(primitive_mode); + + source += "layout (" + std::string(glsl_topology) + ") in;\n"; + source += "#define MAX_VERTEX_INPUT " + std::to_string(max_vertices) + '\n'; + } + + source += code; + + OGLShader shader; + shader.Create(source.c_str(), GetShaderType(program_type)); + + auto program = std::make_shared(); + program->Create(true, hint_retrievable, shader.handle); + return program; +} + +std::set GetSupportedFormats() { + std::set supported_formats; + + GLint num_formats{}; + glGetIntegerv(GL_NUM_PROGRAM_BINARY_FORMATS, &num_formats); + + std::vector formats(num_formats); + glGetIntegerv(GL_PROGRAM_BINARY_FORMATS, formats.data()); + + for (const GLint format : formats) + supported_formats.insert(static_cast(format)); + return supported_formats; +} + +} // namespace + +CachedShader::CachedShader(VAddr addr, u64 unique_identifier, Maxwell::ShaderProgram program_type, + ShaderDiskCacheOpenGL& disk_cache, + const PrecompiledPrograms& precompiled_programs, + ProgramCode&& program_code, ProgramCode&& program_code_b) + : addr{addr}, unique_identifier{unique_identifier}, program_type{program_type}, + disk_cache{disk_cache}, precompiled_programs{precompiled_programs} { + + const std::size_t code_size = CalculateProgramSize(program_code); + const std::size_t code_size_b = + program_code_b.empty() ? 0 : CalculateProgramSize(program_code_b); + + GLShader::ProgramResult program_result = + CreateProgram(program_type, program_code, program_code_b); + if (program_result.first.empty()) { + // TODO(Rodrigo): Unimplemented shader stages hit here, avoid using these for now return; } code = program_result.first; entries = program_result.second; shader_length = entries.shader_length; + + const ShaderDiskCacheRaw raw(unique_identifier, program_type, + static_cast(code_size / sizeof(u64)), + static_cast(code_size_b / sizeof(u64)), + std::move(program_code), std::move(program_code_b)); + disk_cache.SaveRaw(raw); +} + +CachedShader::CachedShader(VAddr addr, u64 unique_identifier, Maxwell::ShaderProgram program_type, + ShaderDiskCacheOpenGL& disk_cache, + const PrecompiledPrograms& precompiled_programs, + GLShader::ProgramResult result) + : addr{addr}, unique_identifier{unique_identifier}, program_type{program_type}, + disk_cache{disk_cache}, precompiled_programs{precompiled_programs} { + + code = std::move(result.first); + entries = result.second; + shader_length = entries.shader_length; } std::tuple CachedShader::GetProgramHandle(GLenum primitive_mode, @@ -94,138 +261,195 @@ std::tuple CachedShader::GetProgramHandle(GLenum primitive const auto [entry, is_cache_miss] = programs.try_emplace(base_bindings); auto& program = entry->second; if (is_cache_miss) { - std::string source = AllocateBindings(base_bindings); - source += code; + program = TryLoadProgram(primitive_mode, base_bindings); + if (!program) { + program = + SpecializeShader(code, entries, program_type, base_bindings, primitive_mode); + disk_cache.SaveUsage(GetUsage(primitive_mode, base_bindings)); + } - OGLShader shader; - shader.Create(source.c_str(), GetShaderType(program_type)); - program.Create(true, shader.handle); - LabelGLObject(GL_PROGRAM, program.handle, addr); + LabelGLObject(GL_PROGRAM, program->handle, addr); } - handle = program.handle; + handle = program->handle; } - // Add const buffer and samplers offset reserved by this shader. One UBO binding is reserved for - // emulation values - base_bindings.cbuf += static_cast(entries.const_buffers.size()) + 1; + base_bindings.cbuf += static_cast(entries.const_buffers.size()) + RESERVED_UBOS; base_bindings.gmem += static_cast(entries.global_memory_entries.size()); base_bindings.sampler += static_cast(entries.samplers.size()); return {handle, base_bindings}; } -std::string CachedShader::AllocateBindings(BaseBindings base_bindings) { - std::string code = "#version 430 core\n"; - code += fmt::format("#define EMULATION_UBO_BINDING {}\n", base_bindings.cbuf++); - - for (const auto& cbuf : entries.const_buffers) { - code += fmt::format("#define CBUF_BINDING_{} {}\n", cbuf.GetIndex(), base_bindings.cbuf++); - } - - for (const auto& gmem : entries.global_memory_entries) { - code += fmt::format("#define GMEM_BINDING_{}_{} {}\n", gmem.GetCbufIndex(), - gmem.GetCbufOffset(), base_bindings.gmem++); - } - - for (const auto& sampler : entries.samplers) { - code += fmt::format("#define SAMPLER_BINDING_{} {}\n", sampler.GetIndex(), - base_bindings.sampler++); - } - - return code; -} - GLuint CachedShader::GetGeometryShader(GLenum primitive_mode, BaseBindings base_bindings) { const auto [entry, is_cache_miss] = geometry_programs.try_emplace(base_bindings); auto& programs = entry->second; switch (primitive_mode) { case GL_POINTS: - return LazyGeometryProgram(programs.points, base_bindings, "points", 1, "ShaderPoints"); + return LazyGeometryProgram(programs.points, base_bindings, primitive_mode); case GL_LINES: case GL_LINE_STRIP: - return LazyGeometryProgram(programs.lines, base_bindings, "lines", 2, "ShaderLines"); + return LazyGeometryProgram(programs.lines, base_bindings, primitive_mode); case GL_LINES_ADJACENCY: case GL_LINE_STRIP_ADJACENCY: - return LazyGeometryProgram(programs.lines_adjacency, base_bindings, "lines_adjacency", 4, - "ShaderLinesAdjacency"); + return LazyGeometryProgram(programs.lines_adjacency, base_bindings, primitive_mode); case GL_TRIANGLES: case GL_TRIANGLE_STRIP: case GL_TRIANGLE_FAN: - return LazyGeometryProgram(programs.triangles, base_bindings, "triangles", 3, - "ShaderTriangles"); + return LazyGeometryProgram(programs.triangles, base_bindings, primitive_mode); case GL_TRIANGLES_ADJACENCY: case GL_TRIANGLE_STRIP_ADJACENCY: - return LazyGeometryProgram(programs.triangles_adjacency, base_bindings, - "triangles_adjacency", 6, "ShaderTrianglesAdjacency"); + return LazyGeometryProgram(programs.triangles_adjacency, base_bindings, primitive_mode); default: UNREACHABLE_MSG("Unknown primitive mode."); - return LazyGeometryProgram(programs.points, base_bindings, "points", 1, "ShaderPoints"); + return LazyGeometryProgram(programs.points, base_bindings, primitive_mode); } } -GLuint CachedShader::LazyGeometryProgram(OGLProgram& target_program, BaseBindings base_bindings, - const std::string& glsl_topology, u32 max_vertices, - const std::string& debug_name) { - if (target_program.handle != 0) { - return target_program.handle; +GLuint CachedShader::LazyGeometryProgram(CachedProgram& target_program, BaseBindings base_bindings, + GLenum primitive_mode) { + if (target_program) { + return target_program->handle; + } + const auto [_, debug_name, __] = GetPrimitiveDescription(primitive_mode); + target_program = TryLoadProgram(primitive_mode, base_bindings); + if (!target_program) { + target_program = + SpecializeShader(code, entries, program_type, base_bindings, primitive_mode); + disk_cache.SaveUsage(GetUsage(primitive_mode, base_bindings)); } - std::string source = AllocateBindings(base_bindings); - source += "layout (" + glsl_topology + ") in;\n"; - source += "#define MAX_VERTEX_INPUT " + std::to_string(max_vertices) + '\n'; - source += code; - OGLShader shader; - shader.Create(source.c_str(), GL_GEOMETRY_SHADER); - target_program.Create(true, shader.handle); - LabelGLObject(GL_PROGRAM, target_program.handle, addr, debug_name); - return target_program.handle; + LabelGLObject(GL_PROGRAM, target_program->handle, addr, debug_name); + + return target_program->handle; }; -static bool IsSchedInstruction(std::size_t offset, std::size_t main_offset) { - // sched instructions appear once every 4 instructions. - static constexpr std::size_t SchedPeriod = 4; - const std::size_t absolute_offset = offset - main_offset; - return (absolute_offset % SchedPeriod) == 0; +CachedProgram CachedShader::TryLoadProgram(GLenum primitive_mode, + BaseBindings base_bindings) const { + const auto found = precompiled_programs.find(GetUsage(primitive_mode, base_bindings)); + if (found == precompiled_programs.end()) { + return {}; + } + return found->second; } -static std::size_t CalculateProgramSize(const GLShader::ProgramCode& program) { - constexpr std::size_t start_offset = 10; - std::size_t offset = start_offset; - std::size_t size = start_offset * sizeof(u64); - while (offset < program.size()) { - const u64 inst = program[offset]; - if (!IsSchedInstruction(offset, start_offset)) { - if (inst == 0 || (inst >> 52) == 0x50b) { - break; - } - } - size += sizeof(inst); - offset++; - } - return size; -} - -void CachedShader::CalculateProperties() { - setup.program.real_size = CalculateProgramSize(setup.program.code); - setup.program.real_size_b = 0; - setup.program.unique_identifier = Common::CityHash64( - reinterpret_cast(setup.program.code.data()), setup.program.real_size); - if (program_type == Maxwell::ShaderProgram::VertexA) { - std::size_t seed = 0; - boost::hash_combine(seed, setup.program.unique_identifier); - setup.program.real_size_b = CalculateProgramSize(setup.program.code_b); - const u64 identifier_b = Common::CityHash64( - reinterpret_cast(setup.program.code_b.data()), setup.program.real_size_b); - boost::hash_combine(seed, identifier_b); - setup.program.unique_identifier = static_cast(seed); - } +ShaderDiskCacheUsage CachedShader::GetUsage(GLenum primitive_mode, + BaseBindings base_bindings) const { + return {unique_identifier, base_bindings, primitive_mode}; } ShaderCacheOpenGL::ShaderCacheOpenGL(RasterizerOpenGL& rasterizer) : RasterizerCache{rasterizer} {} -void ShaderCacheOpenGL::LoadDiskCache() {} +void ShaderCacheOpenGL::LoadDiskCache() { + std::vector raws; + std::vector usages; + if (!disk_cache.LoadTransferable(raws, usages)) { + return; + } + + std::vector precompiled = disk_cache.LoadPrecompiled(); + const auto SearchPrecompiled = [&precompiled](const ShaderDiskCacheUsage& usage) { + return std::find_if( + precompiled.begin(), precompiled.end(), + [&usage](const auto& precompiled_entry) { return precompiled_entry.usage == usage; }); + }; + + const std::set supported_formats{GetSupportedFormats()}; + const auto unspecialized{GenerateUnspecializedShaders(raws)}; + + // Build shaders + for (std::size_t i = 0; i < usages.size(); ++i) { + const auto& usage{usages[i]}; + LOG_INFO(Render_OpenGL, "Building shader {:016x} ({} of {})", usage.unique_identifier, + i + 1, usages.size()); + + const auto& unspec{unspecialized.at(usage.unique_identifier)}; + + const auto precompiled_it = SearchPrecompiled(usage); + const bool is_precompiled = precompiled_it != precompiled.end(); + + CachedProgram shader; + if (is_precompiled) { + shader = GeneratePrecompiledProgram(precompiled, *precompiled_it, supported_formats); + } + if (!shader) { + shader = SpecializeShader(unspec.code, unspec.entries, unspec.program_type, + usage.bindings, usage.primitive, true); + } + precompiled_programs.insert({usage, std::move(shader)}); + } + + // TODO(Rodrigo): Do state tracking for transferable shaders and do a dummy draw before + // precompiling them + + for (std::size_t i = 0; i < usages.size(); ++i) { + const auto& usage{usages[i]}; + if (SearchPrecompiled(usage) == precompiled.end()) { + const auto& program = precompiled_programs.at(usage); + disk_cache.SavePrecompiled(usage, program->handle); + } + } +} + +CachedProgram ShaderCacheOpenGL::GeneratePrecompiledProgram( + std::vector& precompiled, + const ShaderDiskCachePrecompiledEntry& precompiled_entry, + const std::set& supported_formats) { + + if (supported_formats.find(precompiled_entry.binary_format) == supported_formats.end()) { + LOG_INFO(Render_OpenGL, "Precompiled cache entry with unsupported format - removing"); + disk_cache.InvalidatePrecompiled(); + precompiled.clear(); + return {}; + } + + CachedProgram shader = std::make_shared(); + shader->handle = glCreateProgram(); + glProgramBinary(shader->handle, precompiled_entry.binary_format, + precompiled_entry.binary.data(), + static_cast(precompiled_entry.binary.size())); + + GLint link_status{}; + glGetProgramiv(shader->handle, GL_LINK_STATUS, &link_status); + if (link_status == GL_FALSE) { + LOG_INFO(Render_OpenGL, "Precompiled cache rejected by the driver - removing"); + disk_cache.InvalidatePrecompiled(); + precompiled.clear(); + + shader.reset(); + } + + return shader; +} + +std::map ShaderCacheOpenGL::GenerateUnspecializedShaders( + const std::vector& raws) { + + std::map unspecialized; + for (const auto& raw : raws) { + const u64 calculated_hash = + GetUniqueIdentifier(raw.GetProgramType(), raw.GetProgramCode(), raw.GetProgramCodeB()); + if (raw.GetUniqueIdentifier() != calculated_hash) { + LOG_ERROR( + Render_OpenGL, + "Invalid hash in entry={:016x} (obtained hash={:016x}) - removing shader cache", + raw.GetUniqueIdentifier(), calculated_hash); + disk_cache.InvalidateTransferable(); + return {}; + } + + auto result = + CreateProgram(raw.GetProgramType(), raw.GetProgramCode(), raw.GetProgramCodeB()); + + precompiled_shaders.insert({raw.GetUniqueIdentifier(), result}); + + unspecialized.insert( + {raw.GetUniqueIdentifier(), + {std::move(result.first), std::move(result.second), raw.GetProgramType()}}); + } + return unspecialized; +} Shader ShaderCacheOpenGL::GetStageProgram(Maxwell::ShaderProgram program) { if (!Core::System::GetInstance().GPU().Maxwell3D().dirty_flags.shaders) { @@ -239,7 +463,23 @@ Shader ShaderCacheOpenGL::GetStageProgram(Maxwell::ShaderProgram program) { if (!shader) { // No shader found - create a new one - shader = std::make_shared(program_addr, program); + ProgramCode program_code = GetShaderCode(program_addr); + ProgramCode program_code_b; + if (program == Maxwell::ShaderProgram::VertexA) { + program_code_b = GetShaderCode(GetShaderAddress(Maxwell::ShaderProgram::VertexB)); + } + const u64 unique_identifier = GetUniqueIdentifier(program, program_code, program_code_b); + + const auto found = precompiled_shaders.find(unique_identifier); + if (found != precompiled_shaders.end()) { + shader = + std::make_shared(program_addr, unique_identifier, program, disk_cache, + precompiled_programs, found->second); + } else { + shader = std::make_shared( + program_addr, unique_identifier, program, disk_cache, precompiled_programs, + std::move(program_code), std::move(program_code_b)); + } Register(shader); } diff --git a/src/video_core/renderer_opengl/gl_shader_cache.h b/src/video_core/renderer_opengl/gl_shader_cache.h index 18fb80bcc..763a47bce 100644 --- a/src/video_core/renderer_opengl/gl_shader_cache.h +++ b/src/video_core/renderer_opengl/gl_shader_cache.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -23,13 +24,25 @@ namespace OpenGL { class CachedShader; class RasterizerOpenGL; +struct UnspecializedShader; using Shader = std::shared_ptr; +using CachedProgram = std::shared_ptr; using Maxwell = Tegra::Engines::Maxwell3D::Regs; +using PrecompiledPrograms = std::map; +using PrecompiledShaders = std::map; class CachedShader final : public RasterizerCacheObject { public: - CachedShader(VAddr addr, Maxwell::ShaderProgram program_type); + explicit CachedShader(VAddr addr, u64 unique_identifier, Maxwell::ShaderProgram program_type, + ShaderDiskCacheOpenGL& disk_cache, + const PrecompiledPrograms& precompiled_programs, + ProgramCode&& program_code, ProgramCode&& program_code_b); + + explicit CachedShader(VAddr addr, u64 unique_identifier, Maxwell::ShaderProgram program_type, + ShaderDiskCacheOpenGL& disk_cache, + const PrecompiledPrograms& precompiled_programs, + GLShader::ProgramResult result); VAddr GetAddr() const override { return addr; @@ -56,33 +69,35 @@ private: // declared by the hardware. Workaround this issue by generating a different shader per input // topology class. struct GeometryPrograms { - OGLProgram points; - OGLProgram lines; - OGLProgram lines_adjacency; - OGLProgram triangles; - OGLProgram triangles_adjacency; + CachedProgram points; + CachedProgram lines; + CachedProgram lines_adjacency; + CachedProgram triangles; + CachedProgram triangles_adjacency; }; - std::string AllocateBindings(BaseBindings base_bindings); - GLuint GetGeometryShader(GLenum primitive_mode, BaseBindings base_bindings); /// Generates a geometry shader or returns one that already exists. - GLuint LazyGeometryProgram(OGLProgram& target_program, BaseBindings base_bindings, - const std::string& glsl_topology, u32 max_vertices, - const std::string& debug_name); + GLuint LazyGeometryProgram(CachedProgram& target_program, BaseBindings base_bindings, + GLenum primitive_mode); - void CalculateProperties(); + CachedProgram TryLoadProgram(GLenum primitive_mode, BaseBindings base_bindings) const; + + ShaderDiskCacheUsage GetUsage(GLenum primitive_mode, BaseBindings base_bindings) const; + + const VAddr addr; + const u64 unique_identifier; + const Maxwell::ShaderProgram program_type; + ShaderDiskCacheOpenGL& disk_cache; + const PrecompiledPrograms& precompiled_programs; - VAddr addr{}; std::size_t shader_length{}; - Maxwell::ShaderProgram program_type{}; - GLShader::ShaderSetup setup; GLShader::ShaderEntries entries; std::string code; - std::map programs; + std::map programs; std::map geometry_programs; std::map cbuf_resource_cache; @@ -101,7 +116,19 @@ public: Shader GetStageProgram(Maxwell::ShaderProgram program); private: + std::map GenerateUnspecializedShaders( + const std::vector& raws); + + CachedProgram GeneratePrecompiledProgram( + std::vector& precompiled, + const ShaderDiskCachePrecompiledEntry& precompiled_entry, + const std::set& supported_formats); + std::array last_shaders; + + ShaderDiskCacheOpenGL disk_cache; + PrecompiledShaders precompiled_shaders; + PrecompiledPrograms precompiled_programs; }; } // namespace OpenGL From be4641c43f0c6c68d183549a9a8715ba6fde9c50 Mon Sep 17 00:00:00 2001 From: ReinUsesLisp Date: Mon, 14 Jan 2019 01:58:46 -0300 Subject: [PATCH 14/32] gl_shader_disk_cache: Invalidate shader cache changes with CMake hash --- CMakeLists.txt | 13 --- CMakeModules/GenerateSCMRev.cmake | 101 ++++++++++++++++++ src/common/CMakeLists.txt | 92 +++++++++------- src/common/scm_rev.cpp.in | 2 + src/common/scm_rev.h | 1 + .../renderer_opengl/gl_shader_disk_cache.cpp | 23 ++-- 6 files changed, 173 insertions(+), 59 deletions(-) create mode 100644 CMakeModules/GenerateSCMRev.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 871e0ca1a..97d888762 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -419,19 +419,6 @@ function(create_target_directory_groups target_name) endforeach() endfunction() -# Gets a UTC timstamp and sets the provided variable to it -function(get_timestamp _var) - string(TIMESTAMP timestamp UTC) - set(${_var} "${timestamp}" PARENT_SCOPE) -endfunction() - -# generate git/build information -include(GetGitRevisionDescription) -get_git_head_revision(GIT_REF_SPEC GIT_REV) -git_describe(GIT_DESC --always --long --dirty) -git_branch_name(GIT_BRANCH) -get_timestamp(BUILD_DATE) - enable_testing() add_subdirectory(externals) add_subdirectory(src) diff --git a/CMakeModules/GenerateSCMRev.cmake b/CMakeModules/GenerateSCMRev.cmake new file mode 100644 index 000000000..e82ad204d --- /dev/null +++ b/CMakeModules/GenerateSCMRev.cmake @@ -0,0 +1,101 @@ +# Gets a UTC timstamp and sets the provided variable to it +function(get_timestamp _var) + string(TIMESTAMP timestamp UTC) + set(${_var} "${timestamp}" PARENT_SCOPE) +endfunction() + +list(APPEND CMAKE_MODULE_PATH "${SRC_DIR}/externals/cmake-modules") +# generate git/build information +include(GetGitRevisionDescription) +get_git_head_revision(GIT_REF_SPEC GIT_REV) +git_describe(GIT_DESC --always --long --dirty) +git_branch_name(GIT_BRANCH) +get_timestamp(BUILD_DATE) + +# Generate cpp with Git revision from template +# Also if this is a CI build, add the build name (ie: Nightly, Canary) to the scm_rev file as well +set(REPO_NAME "") +set(BUILD_VERSION "0") +if ($ENV{CI}) + if ($ENV{TRAVIS}) + set(BUILD_REPOSITORY $ENV{TRAVIS_REPO_SLUG}) + set(BUILD_TAG $ENV{TRAVIS_TAG}) + elseif($ENV{APPVEYOR}) + set(BUILD_REPOSITORY $ENV{APPVEYOR_REPO_NAME}) + set(BUILD_TAG $ENV{APPVEYOR_REPO_TAG_NAME}) + endif() + # regex capture the string nightly or canary into CMAKE_MATCH_1 + string(REGEX MATCH "citra-emu/citra-?(.*)" OUTVAR ${BUILD_REPOSITORY}) + if (${CMAKE_MATCH_COUNT} GREATER 0) + # capitalize the first letter of each word in the repo name. + string(REPLACE "-" ";" REPO_NAME_LIST ${CMAKE_MATCH_1}) + foreach(WORD ${REPO_NAME_LIST}) + string(SUBSTRING ${WORD} 0 1 FIRST_LETTER) + string(SUBSTRING ${WORD} 1 -1 REMAINDER) + string(TOUPPER ${FIRST_LETTER} FIRST_LETTER) + set(REPO_NAME "${REPO_NAME}${FIRST_LETTER}${REMAINDER}") + endforeach() + if (BUILD_TAG) + string(REGEX MATCH "${CMAKE_MATCH_1}-([0-9]+)" OUTVAR ${BUILD_TAG}) + if (${CMAKE_MATCH_COUNT} GREATER 0) + set(BUILD_VERSION ${CMAKE_MATCH_1}) + endif() + if (BUILD_VERSION) + # This leaves a trailing space on the last word, but we actually want that + # because of how it's styled in the title bar. + set(BUILD_FULLNAME "${REPO_NAME} ${BUILD_VERSION} ") + else() + set(BUILD_FULLNAME "") + endif() + endif() + endif() +endif() + +# The variable SRC_DIR must be passed into the script (since it uses the current build directory for all values of CMAKE_*_DIR) +set(VIDEO_CORE "${SRC_DIR}/src/video_core") +set(HASH_FILES + "${VIDEO_CORE}/renderer_opengl/gl_shader_cache.cpp" + "${VIDEO_CORE}/renderer_opengl/gl_shader_cache.h" + "${VIDEO_CORE}/renderer_opengl/gl_shader_decompiler.cpp" + "${VIDEO_CORE}/renderer_opengl/gl_shader_decompiler.h" + "${VIDEO_CORE}/renderer_opengl/gl_shader_disk_cache.cpp" + "${VIDEO_CORE}/renderer_opengl/gl_shader_disk_cache.h" + "${VIDEO_CORE}/renderer_opengl/gl_shader_gen.cpp" + "${VIDEO_CORE}/renderer_opengl/gl_shader_gen.h" + "${VIDEO_CORE}/shader/decode/arithmetic.cpp" + "${VIDEO_CORE}/shader/decode/arithmetic_half.cpp" + "${VIDEO_CORE}/shader/decode/arithmetic_half_immediate.cpp" + "${VIDEO_CORE}/shader/decode/arithmetic_immediate.cpp" + "${VIDEO_CORE}/shader/decode/arithmetic_integer.cpp" + "${VIDEO_CORE}/shader/decode/arithmetic_integer_immediate.cpp" + "${VIDEO_CORE}/shader/decode/bfe.cpp" + "${VIDEO_CORE}/shader/decode/bfi.cpp" + "${VIDEO_CORE}/shader/decode/conversion.cpp" + "${VIDEO_CORE}/shader/decode/ffma.cpp" + "${VIDEO_CORE}/shader/decode/float_set.cpp" + "${VIDEO_CORE}/shader/decode/float_set_predicate.cpp" + "${VIDEO_CORE}/shader/decode/half_set.cpp" + "${VIDEO_CORE}/shader/decode/half_set_predicate.cpp" + "${VIDEO_CORE}/shader/decode/hfma2.cpp" + "${VIDEO_CORE}/shader/decode/integer_set.cpp" + "${VIDEO_CORE}/shader/decode/integer_set_predicate.cpp" + "${VIDEO_CORE}/shader/decode/memory.cpp" + "${VIDEO_CORE}/shader/decode/other.cpp" + "${VIDEO_CORE}/shader/decode/predicate_set_predicate.cpp" + "${VIDEO_CORE}/shader/decode/predicate_set_register.cpp" + "${VIDEO_CORE}/shader/decode/register_set_predicate.cpp" + "${VIDEO_CORE}/shader/decode/shift.cpp" + "${VIDEO_CORE}/shader/decode/video.cpp" + "${VIDEO_CORE}/shader/decode/xmad.cpp" + "${VIDEO_CORE}/shader/decode.cpp" + "${VIDEO_CORE}/shader/shader_ir.cpp" + "${VIDEO_CORE}/shader/shader_ir.h" + "${VIDEO_CORE}/shader/track.cpp" +) +set(COMBINED "") +foreach (F IN LISTS HASH_FILES) + file(READ ${F} TMP) + set(COMBINED "${COMBINED}${TMP}") +endforeach() +string(MD5 SHADER_CACHE_VERSION "${COMBINED}") +configure_file("${SRC_DIR}/src/common/scm_rev.cpp.in" "scm_rev.cpp" @ONLY) \ No newline at end of file diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 845626fc5..f38c0fee9 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -1,42 +1,56 @@ -# Generate cpp with Git revision from template -# Also if this is a CI build, add the build name (ie: Nightly, Canary) to the scm_rev file as well -set(REPO_NAME "") -set(BUILD_VERSION "0") -if ($ENV{CI}) - if ($ENV{TRAVIS}) - set(BUILD_REPOSITORY $ENV{TRAVIS_REPO_SLUG}) - set(BUILD_TAG $ENV{TRAVIS_TAG}) - elseif($ENV{APPVEYOR}) - set(BUILD_REPOSITORY $ENV{APPVEYOR_REPO_NAME}) - set(BUILD_TAG $ENV{APPVEYOR_REPO_TAG_NAME}) - endif() - # regex capture the string nightly or canary into CMAKE_MATCH_1 - string(REGEX MATCH "yuzu-emu/yuzu-?(.*)" OUTVAR ${BUILD_REPOSITORY}) - if (${CMAKE_MATCH_COUNT} GREATER 0) - # capitalize the first letter of each word in the repo name. - string(REPLACE "-" ";" REPO_NAME_LIST ${CMAKE_MATCH_1}) - foreach(WORD ${REPO_NAME_LIST}) - string(SUBSTRING ${WORD} 0 1 FIRST_LETTER) - string(SUBSTRING ${WORD} 1 -1 REMAINDER) - string(TOUPPER ${FIRST_LETTER} FIRST_LETTER) - set(REPO_NAME "${REPO_NAME}${FIRST_LETTER}${REMAINDER}") - endforeach() - if (BUILD_TAG) - string(REGEX MATCH "${CMAKE_MATCH_1}-([0-9]+)" OUTVAR ${BUILD_TAG}) - if (${CMAKE_MATCH_COUNT} GREATER 0) - set(BUILD_VERSION ${CMAKE_MATCH_1}) - endif() - if (BUILD_VERSION) - # This leaves a trailing space on the last word, but we actually want that - # because of how it's styled in the title bar. - set(BUILD_FULLNAME "${REPO_NAME} ${BUILD_VERSION} ") - else() - set(BUILD_FULLNAME "") - endif() - endif() - endif() -endif() -configure_file("${CMAKE_CURRENT_SOURCE_DIR}/scm_rev.cpp.in" "${CMAKE_CURRENT_SOURCE_DIR}/scm_rev.cpp" @ONLY) +# Add a custom command to generate a new shader_cache_version hash when any of the following files change +# NOTE: This is an approximation of what files affect shader generation, its possible something else +# could affect the result, but much more unlikely than the following files. Keeping a list of files +# like this allows for much better caching since it doesn't force the user to recompile binary shaders every update +set(VIDEO_CORE "${CMAKE_SOURCE_DIR}/src/video_core") +add_custom_command(OUTPUT scm_rev.cpp + COMMAND cmake -DSRC_DIR="${CMAKE_SOURCE_DIR}" -P "${CMAKE_SOURCE_DIR}/CMakeModules/GenerateSCMRev.cmake" + DEPENDS + # WARNING! It was too much work to try and make a common location for this list, + # so if you need to change it, please update CMakeModules/GenerateSCMRev.cmake as well + "${VIDEO_CORE}/renderer_opengl/gl_shader_cache.cpp" + "${VIDEO_CORE}/renderer_opengl/gl_shader_cache.h" + "${VIDEO_CORE}/renderer_opengl/gl_shader_decompiler.cpp" + "${VIDEO_CORE}/renderer_opengl/gl_shader_decompiler.h" + "${VIDEO_CORE}/renderer_opengl/gl_shader_disk_cache.cpp" + "${VIDEO_CORE}/renderer_opengl/gl_shader_disk_cache.h" + "${VIDEO_CORE}/renderer_opengl/gl_shader_gen.cpp" + "${VIDEO_CORE}/renderer_opengl/gl_shader_gen.h" + "${VIDEO_CORE}/shader/decode/arithmetic.cpp" + "${VIDEO_CORE}/shader/decode/arithmetic_half.cpp" + "${VIDEO_CORE}/shader/decode/arithmetic_half_immediate.cpp" + "${VIDEO_CORE}/shader/decode/arithmetic_immediate.cpp" + "${VIDEO_CORE}/shader/decode/arithmetic_integer.cpp" + "${VIDEO_CORE}/shader/decode/arithmetic_integer_immediate.cpp" + "${VIDEO_CORE}/shader/decode/bfe.cpp" + "${VIDEO_CORE}/shader/decode/bfi.cpp" + "${VIDEO_CORE}/shader/decode/conversion.cpp" + "${VIDEO_CORE}/shader/decode/ffma.cpp" + "${VIDEO_CORE}/shader/decode/float_set.cpp" + "${VIDEO_CORE}/shader/decode/float_set_predicate.cpp" + "${VIDEO_CORE}/shader/decode/half_set.cpp" + "${VIDEO_CORE}/shader/decode/half_set_predicate.cpp" + "${VIDEO_CORE}/shader/decode/hfma2.cpp" + "${VIDEO_CORE}/shader/decode/integer_set.cpp" + "${VIDEO_CORE}/shader/decode/integer_set_predicate.cpp" + "${VIDEO_CORE}/shader/decode/memory.cpp" + "${VIDEO_CORE}/shader/decode/other.cpp" + "${VIDEO_CORE}/shader/decode/predicate_set_predicate.cpp" + "${VIDEO_CORE}/shader/decode/predicate_set_register.cpp" + "${VIDEO_CORE}/shader/decode/register_set_predicate.cpp" + "${VIDEO_CORE}/shader/decode/shift.cpp" + "${VIDEO_CORE}/shader/decode/video.cpp" + "${VIDEO_CORE}/shader/decode/xmad.cpp" + "${VIDEO_CORE}/shader/decode.cpp" + "${VIDEO_CORE}/shader/shader_ir.cpp" + "${VIDEO_CORE}/shader/shader_ir.h" + "${VIDEO_CORE}/shader/track.cpp" + # and also check that the scm_rev files haven't changed + "${CMAKE_CURRENT_SOURCE_DIR}/scm_rev.cpp.in" + "${CMAKE_CURRENT_SOURCE_DIR}/scm_rev.h" + # technically we should regenerate if the git version changed, but its not worth the effort imo + "${CMAKE_SOURCE_DIR}/CMakeModules/GenerateSCMRev.cmake" +) add_library(common STATIC alignment.h diff --git a/src/common/scm_rev.cpp.in b/src/common/scm_rev.cpp.in index 2b1727769..d69038f65 100644 --- a/src/common/scm_rev.cpp.in +++ b/src/common/scm_rev.cpp.in @@ -11,6 +11,7 @@ #define BUILD_DATE "@BUILD_DATE@" #define BUILD_FULLNAME "@BUILD_FULLNAME@" #define BUILD_VERSION "@BUILD_VERSION@" +#define SHADER_CACHE_VERSION "@SHADER_CACHE_VERSION@" namespace Common { @@ -21,6 +22,7 @@ const char g_build_name[] = BUILD_NAME; const char g_build_date[] = BUILD_DATE; const char g_build_fullname[] = BUILD_FULLNAME; const char g_build_version[] = BUILD_VERSION; +const char g_shader_cache_version[] = SHADER_CACHE_VERSION; } // namespace diff --git a/src/common/scm_rev.h b/src/common/scm_rev.h index af9a9daed..666bf0367 100644 --- a/src/common/scm_rev.h +++ b/src/common/scm_rev.h @@ -13,5 +13,6 @@ extern const char g_build_name[]; extern const char g_build_date[]; extern const char g_build_fullname[]; extern const char g_build_version[]; +extern const char g_shader_cache_version[]; } // namespace Common diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp index 4d02a800d..4b0e50b90 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp @@ -4,6 +4,8 @@ #pragma once +#include + #include #include "common/assert.h" @@ -11,6 +13,7 @@ #include "common/common_types.h" #include "common/file_util.h" #include "common/logging/log.h" +#include "common/scm_rev.h" #include "core/core.h" #include "core/hle/kernel/process.h" @@ -26,9 +29,7 @@ enum class EntryKind : u32 { }; constexpr u32 NativeVersion = 1; - -// TODO(Rodrigo): Hash files -constexpr u64 PrecompiledHash = 0xdeadbeefdeadbeef; +constexpr u32 ShaderHashSize = 64; // Making sure sizes doesn't change by accident static_assert(sizeof(BaseBindings) == 12); @@ -38,6 +39,12 @@ namespace { std::string GetTitleID() { return fmt::format("{:016X}", Core::CurrentProcess()->GetTitleID()); } + +std::string GetShaderHash() { + std::array hash{}; + std::strncpy(hash.data(), Common::g_shader_cache_version, ShaderHashSize); + return std::string(hash.data()); +} } // namespace ShaderDiskCacheRaw::ShaderDiskCacheRaw(FileUtil::IOFile& file) { @@ -130,9 +137,9 @@ std::vector ShaderDiskCacheOpenGL::LoadPrecompi } const u64 file_size = file.GetSize(); - u64 precompiled_hash{}; - file.ReadBytes(&precompiled_hash, sizeof(precompiled_hash)); - if (precompiled_hash != PrecompiledHash) { + char precompiled_hash[ShaderHashSize]; + file.ReadBytes(&precompiled_hash, ShaderHashSize); + if (std::string(precompiled_hash) != GetShaderHash()) { LOG_INFO(Render_OpenGL, "Precompiled cache is from another version of yuzu - removing"); file.Close(); InvalidatePrecompiled(); @@ -255,7 +262,9 @@ FileUtil::IOFile ShaderDiskCacheOpenGL::AppendPrecompiledFile() const { } if (!existed || file.GetSize() == 0) { - file.WriteObject(PrecompiledHash); + std::array hash{}; + std::strcpy(hash.data(), GetShaderHash().c_str()); + file.WriteArray(hash.data(), hash.size()); } return file; } From e78da8dc1f8b6fbb6e9c1aeff9e54fe7f879d3b1 Mon Sep 17 00:00:00 2001 From: ReinUsesLisp Date: Mon, 14 Jan 2019 02:14:27 -0300 Subject: [PATCH 15/32] settings: Hide shader cache behind a setting --- src/core/settings.h | 1 + src/core/telemetry_session.cpp | 2 ++ .../renderer_opengl/gl_shader_disk_cache.cpp | 21 +++++++++++++++++++ src/yuzu/configuration/config.cpp | 3 +++ src/yuzu/configuration/configure_graphics.cpp | 2 ++ src/yuzu/configuration/configure_graphics.ui | 7 +++++++ src/yuzu_cmd/config.cpp | 2 ++ src/yuzu_cmd/default_ini.h | 4 ++++ 8 files changed, 42 insertions(+) diff --git a/src/core/settings.h b/src/core/settings.h index c97387fc7..7e76e0466 100644 --- a/src/core/settings.h +++ b/src/core/settings.h @@ -391,6 +391,7 @@ struct Values { float resolution_factor; bool use_frame_limit; u16 frame_limit; + bool use_disk_shader_cache; bool use_accurate_gpu_emulation; float bg_red; diff --git a/src/core/telemetry_session.cpp b/src/core/telemetry_session.cpp index 09ed74d78..58dfcc4df 100644 --- a/src/core/telemetry_session.cpp +++ b/src/core/telemetry_session.cpp @@ -158,6 +158,8 @@ TelemetrySession::TelemetrySession() { AddField(Telemetry::FieldType::UserConfig, "Renderer_UseFrameLimit", Settings::values.use_frame_limit); AddField(Telemetry::FieldType::UserConfig, "Renderer_FrameLimit", Settings::values.frame_limit); + AddField(Telemetry::FieldType::UserConfig, "Renderer_UseDiskShaderCache", + Settings::values.use_disk_shader_cache); AddField(Telemetry::FieldType::UserConfig, "Renderer_UseAccurateGpuEmulation", Settings::values.use_accurate_gpu_emulation); AddField(Telemetry::FieldType::UserConfig, "System_UseDockedMode", diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp index 4b0e50b90..6a23b8fe2 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp @@ -17,6 +17,7 @@ #include "core/core.h" #include "core/hle/kernel/process.h" +#include "core/settings.h" #include "video_core/renderer_opengl/gl_shader_cache.h" #include "video_core/renderer_opengl/gl_shader_disk_cache.h" @@ -78,6 +79,10 @@ void ShaderDiskCacheRaw::Save(FileUtil::IOFile& file) const { bool ShaderDiskCacheOpenGL::LoadTransferable(std::vector& raws, std::vector& usages) { + if (!Settings::values.use_disk_shader_cache) { + return false; + } + FileUtil::IOFile file(GetTransferablePath(), "rb"); if (!file.IsOpen()) { LOG_INFO(Render_OpenGL, "No transferable shader cache found for game with title id={}", @@ -129,6 +134,10 @@ bool ShaderDiskCacheOpenGL::LoadTransferable(std::vector& ra } std::vector ShaderDiskCacheOpenGL::LoadPrecompiled() { + if (!Settings::values.use_disk_shader_cache) { + return {}; + } + FileUtil::IOFile file(GetPrecompiledPath(), "rb"); if (!file.IsOpen()) { LOG_INFO(Render_OpenGL, "No precompiled shader cache found for game with title id={}", @@ -173,6 +182,10 @@ void ShaderDiskCacheOpenGL::InvalidatePrecompiled() const { } void ShaderDiskCacheOpenGL::SaveRaw(const ShaderDiskCacheRaw& entry) { + if (!Settings::values.use_disk_shader_cache) { + return; + } + const u64 id = entry.GetUniqueIdentifier(); if (transferable.find(id) != transferable.end()) { // The shader already exists @@ -190,6 +203,10 @@ void ShaderDiskCacheOpenGL::SaveRaw(const ShaderDiskCacheRaw& entry) { } void ShaderDiskCacheOpenGL::SaveUsage(const ShaderDiskCacheUsage& usage) { + if (!Settings::values.use_disk_shader_cache) { + return; + } + const auto it = transferable.find(usage.unique_identifier); if (it == transferable.end()) { LOG_CRITICAL(Render_OpenGL, "Saving shader usage without storing raw previously"); @@ -208,6 +225,10 @@ void ShaderDiskCacheOpenGL::SaveUsage(const ShaderDiskCacheUsage& usage) { } void ShaderDiskCacheOpenGL::SavePrecompiled(const ShaderDiskCacheUsage& usage, GLuint program) { + if (!Settings::values.use_disk_shader_cache) { + return; + } + FileUtil::IOFile file = AppendPrecompiledFile(); if (!file.IsOpen()) { return; diff --git a/src/yuzu/configuration/config.cpp b/src/yuzu/configuration/config.cpp index ddf4cf552..e9546dadf 100644 --- a/src/yuzu/configuration/config.cpp +++ b/src/yuzu/configuration/config.cpp @@ -370,6 +370,8 @@ void Config::ReadValues() { Settings::values.resolution_factor = qt_config->value("resolution_factor", 1.0).toFloat(); Settings::values.use_frame_limit = qt_config->value("use_frame_limit", true).toBool(); Settings::values.frame_limit = qt_config->value("frame_limit", 100).toInt(); + Settings::values.use_disk_shader_cache = + qt_config->value("use_disk_shader_cache", false).toBool(); Settings::values.use_accurate_gpu_emulation = qt_config->value("use_accurate_gpu_emulation", false).toBool(); @@ -629,6 +631,7 @@ void Config::SaveValues() { qt_config->setValue("resolution_factor", (double)Settings::values.resolution_factor); qt_config->setValue("use_frame_limit", Settings::values.use_frame_limit); qt_config->setValue("frame_limit", Settings::values.frame_limit); + qt_config->setValue("use_disk_shader_cache", Settings::values.use_disk_shader_cache); qt_config->setValue("use_accurate_gpu_emulation", Settings::values.use_accurate_gpu_emulation); // Cast to double because Qt's written float values are not human-readable diff --git a/src/yuzu/configuration/configure_graphics.cpp b/src/yuzu/configuration/configure_graphics.cpp index d21f95469..0f5dd534b 100644 --- a/src/yuzu/configuration/configure_graphics.cpp +++ b/src/yuzu/configuration/configure_graphics.cpp @@ -73,6 +73,7 @@ void ConfigureGraphics::setConfiguration() { static_cast(FromResolutionFactor(Settings::values.resolution_factor))); ui->toggle_frame_limit->setChecked(Settings::values.use_frame_limit); ui->frame_limit->setValue(Settings::values.frame_limit); + ui->use_disk_shader_cache->setChecked(Settings::values.use_disk_shader_cache); ui->use_accurate_gpu_emulation->setChecked(Settings::values.use_accurate_gpu_emulation); UpdateBackgroundColorButton(QColor::fromRgbF(Settings::values.bg_red, Settings::values.bg_green, Settings::values.bg_blue)); @@ -83,6 +84,7 @@ void ConfigureGraphics::applyConfiguration() { ToResolutionFactor(static_cast(ui->resolution_factor_combobox->currentIndex())); Settings::values.use_frame_limit = ui->toggle_frame_limit->isChecked(); Settings::values.frame_limit = ui->frame_limit->value(); + Settings::values.use_disk_shader_cache = ui->use_disk_shader_cache->isChecked(); Settings::values.use_accurate_gpu_emulation = ui->use_accurate_gpu_emulation->isChecked(); Settings::values.bg_red = static_cast(bg_color.redF()); Settings::values.bg_green = static_cast(bg_color.greenF()); diff --git a/src/yuzu/configuration/configure_graphics.ui b/src/yuzu/configuration/configure_graphics.ui index e278cdd05..824f5810a 100644 --- a/src/yuzu/configuration/configure_graphics.ui +++ b/src/yuzu/configuration/configure_graphics.ui @@ -49,6 +49,13 @@ + + + + Use disk shader cache + + + diff --git a/src/yuzu_cmd/config.cpp b/src/yuzu_cmd/config.cpp index 7a77f76e8..ff05b3179 100644 --- a/src/yuzu_cmd/config.cpp +++ b/src/yuzu_cmd/config.cpp @@ -350,6 +350,8 @@ void Config::ReadValues() { Settings::values.use_frame_limit = sdl2_config->GetBoolean("Renderer", "use_frame_limit", true); Settings::values.frame_limit = static_cast(sdl2_config->GetInteger("Renderer", "frame_limit", 100)); + Settings::values.use_disk_shader_cache = + sdl2_config->GetBoolean("Renderer", "use_disk_shader_cache", false); Settings::values.use_accurate_gpu_emulation = sdl2_config->GetBoolean("Renderer", "use_accurate_gpu_emulation", false); diff --git a/src/yuzu_cmd/default_ini.h b/src/yuzu_cmd/default_ini.h index ba51a4a51..a81986f8e 100644 --- a/src/yuzu_cmd/default_ini.h +++ b/src/yuzu_cmd/default_ini.h @@ -110,6 +110,10 @@ use_frame_limit = # 1 - 9999: Speed limit as a percentage of target game speed. 100 (default) frame_limit = +# Whether to use disk based shader cache +# 0 (default): Off, 1 : On +use_disk_shader_cache = + # Whether to use accurate GPU emulation # 0 (default): Off (fast), 1 : On (slow) use_accurate_gpu_emulation = From cfb20c4c9d863698b938aaad3d27cfe8e4eedb2b Mon Sep 17 00:00:00 2001 From: ReinUsesLisp Date: Tue, 15 Jan 2019 01:07:57 -0300 Subject: [PATCH 16/32] gl_shader_disk_cache: Save GLSL and entries into the precompiled file --- .../renderer_opengl/gl_rasterizer.cpp | 8 +- .../renderer_opengl/gl_shader_cache.cpp | 71 +++---- .../renderer_opengl/gl_shader_cache.h | 9 +- .../renderer_opengl/gl_shader_decompiler.cpp | 7 +- .../renderer_opengl/gl_shader_decompiler.h | 65 +------ .../renderer_opengl/gl_shader_disk_cache.cpp | 177 +++++++++++++++--- .../renderer_opengl/gl_shader_disk_cache.h | 21 ++- .../renderer_opengl/gl_shader_gen.h | 2 - src/video_core/shader/shader_ir.h | 9 + 9 files changed, 234 insertions(+), 135 deletions(-) diff --git a/src/video_core/renderer_opengl/gl_rasterizer.cpp b/src/video_core/renderer_opengl/gl_rasterizer.cpp index 18aafe767..48e003fa1 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer.cpp +++ b/src/video_core/renderer_opengl/gl_rasterizer.cpp @@ -1009,22 +1009,20 @@ void RasterizerOpenGL::SetupTextures(Maxwell::ShaderStage stage, const Shader& s for (u32 bindpoint = 0; bindpoint < entries.size(); ++bindpoint) { const auto& entry = entries[bindpoint]; + const auto texture = maxwell3d.GetStageTexture(stage, entry.GetOffset()); const u32 current_bindpoint = base_bindings.sampler + bindpoint; - auto& unit = state.texture_units[current_bindpoint]; - - const auto texture = maxwell3d.GetStageTexture(entry.GetStage(), entry.GetOffset()); texture_samplers[current_bindpoint].SyncWithConfig(texture.tsc); Surface surface = res_cache.GetTextureSurface(texture, entry); if (surface != nullptr) { - unit.texture = + state.texture_units[current_bindpoint].texture = entry.IsArray() ? surface->TextureLayer().handle : surface->Texture().handle; surface->UpdateSwizzle(texture.tic.x_source, texture.tic.y_source, texture.tic.z_source, texture.tic.w_source); } else { // Can occur when texture addr is null or its memory is unmapped/invalid - unit.texture = 0; + state.texture_units[current_bindpoint].texture = 0; } } } diff --git a/src/video_core/renderer_opengl/gl_shader_cache.cpp b/src/video_core/renderer_opengl/gl_shader_cache.cpp index a70ff79d0..761b355e4 100644 --- a/src/video_core/renderer_opengl/gl_shader_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_cache.cpp @@ -143,6 +143,8 @@ GLShader::ProgramResult CreateProgram(Maxwell::ShaderProgram program_type, Progr // stage here. setup.SetProgramB(std::move(program_code_b)); } + setup.program.unique_identifier = + GetUniqueIdentifier(program_type, program_code, program_code_b); switch (program_type) { case Maxwell::ShaderProgram::VertexA: @@ -348,15 +350,12 @@ void ShaderCacheOpenGL::LoadDiskCache() { return; } - std::vector precompiled = disk_cache.LoadPrecompiled(); - const auto SearchPrecompiled = [&precompiled](const ShaderDiskCacheUsage& usage) { - return std::find_if( - precompiled.begin(), precompiled.end(), - [&usage](const auto& precompiled_entry) { return precompiled_entry.usage == usage; }); - }; + std::map decompiled; + std::map dumps; + disk_cache.LoadPrecompiled(decompiled, dumps); const std::set supported_formats{GetSupportedFormats()}; - const auto unspecialized{GenerateUnspecializedShaders(raws)}; + const auto unspecialized{GenerateUnspecializedShaders(raws, decompiled)}; // Build shaders for (std::size_t i = 0; i < usages.size(); ++i) { @@ -365,13 +364,17 @@ void ShaderCacheOpenGL::LoadDiskCache() { i + 1, usages.size()); const auto& unspec{unspecialized.at(usage.unique_identifier)}; - - const auto precompiled_it = SearchPrecompiled(usage); - const bool is_precompiled = precompiled_it != precompiled.end(); + const auto dump_it = dumps.find(usage); CachedProgram shader; - if (is_precompiled) { - shader = GeneratePrecompiledProgram(precompiled, *precompiled_it, supported_formats); + if (dump_it != dumps.end()) { + // If the shader is dumped, attempt to load it with + shader = GeneratePrecompiledProgram(dump_it->second, supported_formats); + if (!shader) { + // Invalidate the precompiled cache if a shader dumped shader was rejected + disk_cache.InvalidatePrecompiled(); + dumps.clear(); + } } if (!shader) { shader = SpecializeShader(unspec.code, unspec.entries, unspec.program_type, @@ -385,52 +388,47 @@ void ShaderCacheOpenGL::LoadDiskCache() { for (std::size_t i = 0; i < usages.size(); ++i) { const auto& usage{usages[i]}; - if (SearchPrecompiled(usage) == precompiled.end()) { + if (dumps.find(usage) == dumps.end()) { const auto& program = precompiled_programs.at(usage); - disk_cache.SavePrecompiled(usage, program->handle); + disk_cache.SaveDump(usage, program->handle); } } } CachedProgram ShaderCacheOpenGL::GeneratePrecompiledProgram( - std::vector& precompiled, - const ShaderDiskCachePrecompiledEntry& precompiled_entry, - const std::set& supported_formats) { + const ShaderDiskCacheDump& dump, const std::set& supported_formats) { - if (supported_formats.find(precompiled_entry.binary_format) == supported_formats.end()) { + if (supported_formats.find(dump.binary_format) == supported_formats.end()) { LOG_INFO(Render_OpenGL, "Precompiled cache entry with unsupported format - removing"); - disk_cache.InvalidatePrecompiled(); - precompiled.clear(); return {}; } CachedProgram shader = std::make_shared(); shader->handle = glCreateProgram(); - glProgramBinary(shader->handle, precompiled_entry.binary_format, - precompiled_entry.binary.data(), - static_cast(precompiled_entry.binary.size())); + glProgramBinary(shader->handle, dump.binary_format, dump.binary.data(), + static_cast(dump.binary.size())); GLint link_status{}; glGetProgramiv(shader->handle, GL_LINK_STATUS, &link_status); if (link_status == GL_FALSE) { LOG_INFO(Render_OpenGL, "Precompiled cache rejected by the driver - removing"); - disk_cache.InvalidatePrecompiled(); - precompiled.clear(); - - shader.reset(); + return {}; } return shader; } std::map ShaderCacheOpenGL::GenerateUnspecializedShaders( - const std::vector& raws) { + const std::vector& raws, + const std::map& decompiled) { std::map unspecialized; + for (const auto& raw : raws) { + const u64 unique_identifier = raw.GetUniqueIdentifier(); const u64 calculated_hash = GetUniqueIdentifier(raw.GetProgramType(), raw.GetProgramCode(), raw.GetProgramCodeB()); - if (raw.GetUniqueIdentifier() != calculated_hash) { + if (unique_identifier != calculated_hash) { LOG_ERROR( Render_OpenGL, "Invalid hash in entry={:016x} (obtained hash={:016x}) - removing shader cache", @@ -439,10 +437,19 @@ std::map ShaderCacheOpenGL::GenerateUnspecializedShade return {}; } - auto result = - CreateProgram(raw.GetProgramType(), raw.GetProgramCode(), raw.GetProgramCodeB()); + GLShader::ProgramResult result; + if (const auto it = decompiled.find(unique_identifier); it != decompiled.end()) { + // If it's stored in the precompiled file, avoid decompiling it here + const auto& stored_decompiled{it->second}; + result = {stored_decompiled.code, stored_decompiled.entries}; + } else { + // Otherwise decompile the shader at boot and save the result to the decompiled file + result = + CreateProgram(raw.GetProgramType(), raw.GetProgramCode(), raw.GetProgramCodeB()); + disk_cache.SaveDecompiled(unique_identifier, result.first, result.second); + } - precompiled_shaders.insert({raw.GetUniqueIdentifier(), result}); + precompiled_shaders.insert({unique_identifier, result}); unspecialized.insert( {raw.GetUniqueIdentifier(), diff --git a/src/video_core/renderer_opengl/gl_shader_cache.h b/src/video_core/renderer_opengl/gl_shader_cache.h index 763a47bce..3b5a82f8a 100644 --- a/src/video_core/renderer_opengl/gl_shader_cache.h +++ b/src/video_core/renderer_opengl/gl_shader_cache.h @@ -117,12 +117,11 @@ public: private: std::map GenerateUnspecializedShaders( - const std::vector& raws); + const std::vector& raws, + const std::map& decompiled); - CachedProgram GeneratePrecompiledProgram( - std::vector& precompiled, - const ShaderDiskCachePrecompiledEntry& precompiled_entry, - const std::set& supported_formats); + CachedProgram GeneratePrecompiledProgram(const ShaderDiskCacheDump& dump, + const std::set& supported_formats); std::array last_shaders; diff --git a/src/video_core/renderer_opengl/gl_shader_decompiler.cpp b/src/video_core/renderer_opengl/gl_shader_decompiler.cpp index 9184a1287..d84caa6db 100644 --- a/src/video_core/renderer_opengl/gl_shader_decompiler.cpp +++ b/src/video_core/renderer_opengl/gl_shader_decompiler.cpp @@ -193,13 +193,14 @@ public: ShaderEntries GetShaderEntries() const { ShaderEntries entries; for (const auto& cbuf : ir.GetConstantBuffers()) { - entries.const_buffers.emplace_back(cbuf.second, stage, cbuf.first); + entries.const_buffers.emplace_back(cbuf.second.GetMaxOffset(), cbuf.second.IsIndirect(), + cbuf.first); } for (const auto& sampler : ir.GetSamplers()) { - entries.samplers.emplace_back(sampler, stage); + entries.samplers.emplace_back(sampler); } for (const auto& gmem : ir.GetGlobalMemoryBases()) { - entries.global_memory_entries.emplace_back(gmem.cbuf_index, gmem.cbuf_offset, stage); + entries.global_memory_entries.emplace_back(gmem.cbuf_index, gmem.cbuf_offset); } entries.clip_distances = ir.GetClipDistances(); entries.shader_length = ir.GetLength(); diff --git a/src/video_core/renderer_opengl/gl_shader_decompiler.h b/src/video_core/renderer_opengl/gl_shader_decompiler.h index 398be13d6..0031cb614 100644 --- a/src/video_core/renderer_opengl/gl_shader_decompiler.h +++ b/src/video_core/renderer_opengl/gl_shader_decompiler.h @@ -23,40 +23,23 @@ using Maxwell = Tegra::Engines::Maxwell3D::Regs; class ConstBufferEntry : public VideoCommon::Shader::ConstBuffer { public: - explicit ConstBufferEntry(const VideoCommon::Shader::ConstBuffer& entry, - Maxwell::ShaderStage stage, u32 index) - : VideoCommon::Shader::ConstBuffer{entry}, stage{stage}, index{index} {} - - Maxwell::ShaderStage GetStage() const { - return stage; - } + explicit ConstBufferEntry(u32 max_offset, bool is_indirect, u32 index) + : VideoCommon::Shader::ConstBuffer{max_offset, is_indirect}, index{index} {} u32 GetIndex() const { return index; } private: - Maxwell::ShaderStage stage{}; u32 index{}; }; -class SamplerEntry : public VideoCommon::Shader::Sampler { -public: - explicit SamplerEntry(const VideoCommon::Shader::Sampler& entry, Maxwell::ShaderStage stage) - : VideoCommon::Shader::Sampler{entry}, stage{stage} {} - - Maxwell::ShaderStage GetStage() const { - return stage; - } - -private: - Maxwell::ShaderStage stage{}; -}; +using SamplerEntry = VideoCommon::Shader::Sampler; class GlobalMemoryEntry { public: - explicit GlobalMemoryEntry(u32 cbuf_index, u32 cbuf_offset, Maxwell::ShaderStage stage) - : cbuf_index{cbuf_index}, cbuf_offset{cbuf_offset}, stage{stage} {} + explicit GlobalMemoryEntry(u32 cbuf_index, u32 cbuf_offset) + : cbuf_index{cbuf_index}, cbuf_offset{cbuf_offset} {} u32 GetCbufIndex() const { return cbuf_index; @@ -66,47 +49,9 @@ public: return cbuf_offset; } - Maxwell::ShaderStage GetStage() const { - return stage; - } - private: u32 cbuf_index{}; u32 cbuf_offset{}; - Maxwell::ShaderStage stage{}; - std::string name; -}; - -class GlobalMemoryEntry { -public: - explicit GlobalMemoryEntry(u32 cbuf_index, u32 cbuf_offset, Maxwell::ShaderStage stage, - std::string name) - : cbuf_index{cbuf_index}, cbuf_offset{cbuf_offset}, stage{stage}, name{std::move(name)} {} - - u32 GetCbufIndex() const { - return cbuf_index; - } - - u32 GetCbufOffset() const { - return cbuf_offset; - } - - const std::string& GetName() const { - return name; - } - - Maxwell::ShaderStage GetStage() const { - return stage; - } - - u32 GetHash() const { - return (static_cast(stage) << 24) | (cbuf_index << 16) | cbuf_offset; - } - -private: - u32 cbuf_index{}; - u32 cbuf_offset{}; - Maxwell::ShaderStage stage{}; }; struct ShaderEntries { diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp index 6a23b8fe2..7628a74c2 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp @@ -24,11 +24,16 @@ namespace OpenGL { -enum class EntryKind : u32 { +enum class TransferableEntryKind : u32 { Raw, Usage, }; +enum class PrecompiledEntryKind : u32 { + Decompiled, + Dump, +}; + constexpr u32 NativeVersion = 1; constexpr u32 ShaderHashSize = 64; @@ -108,17 +113,17 @@ bool ShaderDiskCacheOpenGL::LoadTransferable(std::vector& ra // Version is valid, load the shaders while (file.Tell() < file_size) { - EntryKind kind{}; + TransferableEntryKind kind{}; file.ReadBytes(&kind, sizeof(u32)); switch (kind) { - case EntryKind::Raw: { + case TransferableEntryKind::Raw: { ShaderDiskCacheRaw entry{file}; transferable.insert({entry.GetUniqueIdentifier(), {}}); raws.push_back(std::move(entry)); break; } - case EntryKind::Usage: { + case TransferableEntryKind::Usage: { ShaderDiskCacheUsage usage{}; file.ReadBytes(&usage, sizeof(usage)); usages.push_back(std::move(usage)); @@ -133,16 +138,19 @@ bool ShaderDiskCacheOpenGL::LoadTransferable(std::vector& ra return true; } -std::vector ShaderDiskCacheOpenGL::LoadPrecompiled() { +bool ShaderDiskCacheOpenGL::LoadPrecompiled( + std::map& decompiled, + std::map& dumps) { + if (!Settings::values.use_disk_shader_cache) { - return {}; + return false; } FileUtil::IOFile file(GetPrecompiledPath(), "rb"); if (!file.IsOpen()) { LOG_INFO(Render_OpenGL, "No precompiled shader cache found for game with title id={}", GetTitleID()); - return {}; + return false; } const u64 file_size = file.GetSize(); @@ -152,24 +160,102 @@ std::vector ShaderDiskCacheOpenGL::LoadPrecompi LOG_INFO(Render_OpenGL, "Precompiled cache is from another version of yuzu - removing"); file.Close(); InvalidatePrecompiled(); - return {}; + return false; } - std::vector precompiled; while (file.Tell() < file_size) { - ShaderDiskCachePrecompiledEntry entry; - file.ReadBytes(&entry.usage, sizeof(entry.usage)); + PrecompiledEntryKind kind{}; + file.ReadBytes(&kind, sizeof(u32)); - file.ReadBytes(&entry.binary_format, sizeof(u32)); + switch (kind) { + case PrecompiledEntryKind::Decompiled: { + ShaderDiskCacheDecompiled entry; - u32 binary_length{}; - file.ReadBytes(&binary_length, sizeof(u32)); - entry.binary.resize(binary_length); - file.ReadBytes(entry.binary.data(), entry.binary.size()); + u64 unique_identifier{}; + file.ReadBytes(&unique_identifier, sizeof(u64)); - precompiled.push_back(entry); + u32 code_size{}; + file.ReadBytes(&code_size, sizeof(u32)); + std::vector code(code_size); + file.ReadArray(code.data(), code.size()); + entry.code = std::string(reinterpret_cast(code.data()), code_size); + + u32 const_buffers_count{}; + file.ReadBytes(&const_buffers_count, sizeof(u32)); + for (u32 i = 0; i < const_buffers_count; ++i) { + u32 max_offset{}, index{}; + u8 is_indirect{}; + file.ReadBytes(&max_offset, sizeof(u32)); + file.ReadBytes(&index, sizeof(u32)); + file.ReadBytes(&is_indirect, sizeof(u8)); + + entry.entries.const_buffers.emplace_back(max_offset, is_indirect != 0, index); + } + + u32 samplers_count{}; + file.ReadBytes(&samplers_count, sizeof(u32)); + for (u32 i = 0; i < samplers_count; ++i) { + u64 offset{}, index{}; + u32 type{}; + u8 is_array{}, is_shadow{}; + file.ReadBytes(&offset, sizeof(u64)); + file.ReadBytes(&index, sizeof(u64)); + file.ReadBytes(&type, sizeof(u32)); + file.ReadBytes(&is_array, sizeof(u8)); + file.ReadBytes(&is_shadow, sizeof(u8)); + + entry.entries.samplers.emplace_back( + static_cast(offset), static_cast(index), + static_cast(type), is_array != 0, is_shadow != 0); + } + + u32 global_memory_count{}; + file.ReadBytes(&global_memory_count, sizeof(u32)); + for (u32 i = 0; i < global_memory_count; ++i) { + u32 cbuf_index{}, cbuf_offset{}; + file.ReadBytes(&cbuf_index, sizeof(u32)); + file.ReadBytes(&cbuf_offset, sizeof(u32)); + entry.entries.global_memory_entries.emplace_back(cbuf_index, cbuf_offset); + } + + for (auto& clip_distance : entry.entries.clip_distances) { + u8 clip_distance_raw{}; + file.ReadBytes(&clip_distance_raw, sizeof(u8)); + clip_distance = clip_distance_raw != 0; + } + + u64 shader_length{}; + file.ReadBytes(&shader_length, sizeof(u64)); + entry.entries.shader_length = static_cast(shader_length); + + decompiled.insert({unique_identifier, std::move(entry)}); + break; + } + case PrecompiledEntryKind::Dump: { + ShaderDiskCacheUsage usage; + file.ReadBytes(&usage, sizeof(usage)); + + ShaderDiskCacheDump dump; + file.ReadBytes(&dump.binary_format, sizeof(u32)); + + u32 binary_length{}; + file.ReadBytes(&binary_length, sizeof(u32)); + dump.binary.resize(binary_length); + file.ReadBytes(dump.binary.data(), dump.binary.size()); + + dumps.insert({usage, dump}); + break; + } + default: + LOG_ERROR(Render_OpenGL, "Unknown precompiled shader cache entry kind={} - removing", + static_cast(kind)); + InvalidatePrecompiled(); + dumps.clear(); + decompiled.clear(); + return false; + } } - return precompiled; + return true; } void ShaderDiskCacheOpenGL::InvalidateTransferable() const { @@ -196,7 +282,7 @@ void ShaderDiskCacheOpenGL::SaveRaw(const ShaderDiskCacheRaw& entry) { if (!file.IsOpen()) { return; } - file.WriteObject(EntryKind::Raw); + file.WriteObject(TransferableEntryKind::Raw); entry.Save(file); transferable.insert({id, {}}); @@ -220,11 +306,12 @@ void ShaderDiskCacheOpenGL::SaveUsage(const ShaderDiskCacheUsage& usage) { if (!file.IsOpen()) { return; } - file.WriteObject(EntryKind::Usage); + file.WriteObject(TransferableEntryKind::Usage); file.WriteObject(usage); } -void ShaderDiskCacheOpenGL::SavePrecompiled(const ShaderDiskCacheUsage& usage, GLuint program) { +void ShaderDiskCacheOpenGL::SaveDecompiled(u64 unique_identifier, const std::string& code, + const GLShader::ShaderEntries& entries) { if (!Settings::values.use_disk_shader_cache) { return; } @@ -234,6 +321,54 @@ void ShaderDiskCacheOpenGL::SavePrecompiled(const ShaderDiskCacheUsage& usage, G return; } + file.WriteObject(static_cast(PrecompiledEntryKind::Decompiled)); + + file.WriteObject(unique_identifier); + + file.WriteObject(static_cast(code.size())); + file.WriteArray(code.data(), code.size()); + + file.WriteObject(static_cast(entries.const_buffers.size())); + for (const auto& cbuf : entries.const_buffers) { + file.WriteObject(static_cast(cbuf.GetMaxOffset())); + file.WriteObject(static_cast(cbuf.GetIndex())); + file.WriteObject(static_cast(cbuf.IsIndirect() ? 1 : 0)); + } + + file.WriteObject(static_cast(entries.samplers.size())); + for (const auto& sampler : entries.samplers) { + file.WriteObject(static_cast(sampler.GetOffset())); + file.WriteObject(static_cast(sampler.GetIndex())); + file.WriteObject(static_cast(sampler.GetType())); + file.WriteObject(static_cast(sampler.IsArray() ? 1 : 0)); + file.WriteObject(static_cast(sampler.IsShadow() ? 1 : 0)); + } + + file.WriteObject(static_cast(entries.global_memory_entries.size())); + for (const auto& gmem : entries.global_memory_entries) { + file.WriteObject(static_cast(gmem.GetCbufIndex())); + file.WriteObject(static_cast(gmem.GetCbufOffset())); + } + + for (const bool clip_distance : entries.clip_distances) { + file.WriteObject(static_cast(clip_distance ? 1 : 0)); + } + + file.WriteObject(static_cast(entries.shader_length)); +} + +void ShaderDiskCacheOpenGL::SaveDump(const ShaderDiskCacheUsage& usage, GLuint program) { + if (!Settings::values.use_disk_shader_cache) { + return; + } + + FileUtil::IOFile file = AppendPrecompiledFile(); + if (!file.IsOpen()) { + return; + } + + file.WriteObject(static_cast(PrecompiledEntryKind::Dump)); + file.WriteObject(usage); GLint binary_length{}; diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.h b/src/video_core/renderer_opengl/gl_shader_disk_cache.h index 6c4c7bd5c..f11693789 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.h +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.h @@ -130,14 +130,16 @@ public: } }; -struct ShaderDiskCachePrecompiledEntry { - ShaderDiskCacheUsage usage; - GLenum binary_format; - std::vector binary; +struct ShaderDiskCacheDecompiled { std::string code; GLShader::ShaderEntries entries; }; +struct ShaderDiskCacheDump { + GLenum binary_format; + std::vector binary; +}; + class ShaderDiskCacheOpenGL { public: /// Loads transferable cache. If file has a old version, it deletes it. Returns true on success. @@ -145,7 +147,8 @@ public: std::vector& usages); /// Loads current game's precompiled cache. Invalidates if emulator's version has changed. - std::vector LoadPrecompiled(); + bool LoadPrecompiled(std::map& decompiled, + std::map& dumps); /// Removes the transferable (and precompiled) cache file. void InvalidateTransferable() const; @@ -159,8 +162,12 @@ public: /// Saves shader usage to the transferable file. Does not check for collisions. void SaveUsage(const ShaderDiskCacheUsage& usage); - /// Saves a precompiled shader entry. Does not check for collisions. - void SavePrecompiled(const ShaderDiskCacheUsage& usage, GLuint program); + /// Saves a decompiled entry to the precompiled file. Does not check for collisions. + void SaveDecompiled(u64 unique_identifier, const std::string& code, + const GLShader::ShaderEntries& entries); + + /// Saves a dump entry to the precompiled file. Does not check for collisions. + void SaveDump(const ShaderDiskCacheUsage& usage, GLuint program); private: /// Opens current game's transferable file and write it's header if it doesn't exist diff --git a/src/video_core/renderer_opengl/gl_shader_gen.h b/src/video_core/renderer_opengl/gl_shader_gen.h index ac5e6917b..fd3105de3 100644 --- a/src/video_core/renderer_opengl/gl_shader_gen.h +++ b/src/video_core/renderer_opengl/gl_shader_gen.h @@ -26,8 +26,6 @@ struct ShaderSetup { ProgramCode code; ProgramCode code_b; // Used for dual vertex shaders u64 unique_identifier; - std::size_t real_size; - std::size_t real_size_b; } program; /// Used in scenarios where we have a dual vertex shaders diff --git a/src/video_core/shader/shader_ir.h b/src/video_core/shader/shader_ir.h index 6e42e3dfb..ef0f3a106 100644 --- a/src/video_core/shader/shader_ir.h +++ b/src/video_core/shader/shader_ir.h @@ -236,6 +236,11 @@ private: class ConstBuffer { public: + explicit ConstBuffer(u32 max_offset, bool is_indirect) + : max_offset{max_offset}, is_indirect{is_indirect} {} + + ConstBuffer() = default; + void MarkAsUsed(u64 offset) { max_offset = std::max(max_offset, static_cast(offset)); } @@ -252,6 +257,10 @@ public: return max_offset + sizeof(float); } + u32 GetMaxOffset() const { + return max_offset; + } + private: u32 max_offset{}; bool is_indirect{}; From f087639e4a0b7ff5fbe06fe84f78302f89fae1a6 Mon Sep 17 00:00:00 2001 From: ReinUsesLisp Date: Tue, 15 Jan 2019 01:48:29 -0300 Subject: [PATCH 17/32] gl_shader_disk_cache: Compress GLSL code using LZ4 --- src/video_core/CMakeLists.txt | 2 +- .../renderer_opengl/gl_shader_disk_cache.cpp | 61 +++++++++++++++++-- 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt index 48be37082..33e507e69 100644 --- a/src/video_core/CMakeLists.txt +++ b/src/video_core/CMakeLists.txt @@ -104,4 +104,4 @@ add_library(video_core STATIC create_target_directory_groups(video_core) target_link_libraries(video_core PUBLIC common core) -target_link_libraries(video_core PRIVATE glad) +target_link_libraries(video_core PRIVATE glad lz4_static) diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp index 7628a74c2..bea4b29d1 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp @@ -5,8 +5,8 @@ #pragma once #include - #include +#include #include "common/assert.h" #include "common/common_paths.h" @@ -51,6 +51,35 @@ std::string GetShaderHash() { std::strncpy(hash.data(), Common::g_shader_cache_version, ShaderHashSize); return std::string(hash.data()); } + +template +std::vector CompressData(const T* source, std::size_t source_size) { + const auto source_size_int = static_cast(source_size); + const int max_compressed_size = LZ4_compressBound(source_size_int); + std::vector compressed(max_compressed_size); + const int compressed_size = LZ4_compress_default(reinterpret_cast(source), + reinterpret_cast(compressed.data()), + source_size_int, max_compressed_size); + if (compressed_size < 0) { + // Compression failed + return {}; + } + compressed.resize(compressed_size); + return compressed; +} + +std::vector DecompressData(const std::vector& compressed, std::size_t uncompressed_size) { + std::vector uncompressed(uncompressed_size); + const int size_check = LZ4_decompress_safe(reinterpret_cast(compressed.data()), + reinterpret_cast(uncompressed.data()), + static_cast(compressed.size()), + static_cast(uncompressed.size())); + if (static_cast(uncompressed_size) != size_check) { + // Decompression failed + return {}; + } + return uncompressed; +} } // namespace ShaderDiskCacheRaw::ShaderDiskCacheRaw(FileUtil::IOFile& file) { @@ -175,10 +204,24 @@ bool ShaderDiskCacheOpenGL::LoadPrecompiled( file.ReadBytes(&unique_identifier, sizeof(u64)); u32 code_size{}; + u32 compressed_code_size{}; file.ReadBytes(&code_size, sizeof(u32)); - std::vector code(code_size); - file.ReadArray(code.data(), code.size()); - entry.code = std::string(reinterpret_cast(code.data()), code_size); + file.ReadBytes(&compressed_code_size, sizeof(u32)); + + std::vector compressed_code(compressed_code_size); + file.ReadArray(compressed_code.data(), compressed_code.size()); + + const std::vector code = DecompressData(compressed_code, code_size); + if (code.empty()) { + LOG_ERROR(Render_OpenGL, + "Failed to decompress GLSL code in precompiled shader={:016x} - removing", + unique_identifier); + InvalidatePrecompiled(); + dumps.clear(); + decompiled.clear(); + return false; + } + entry.code = std::string(reinterpret_cast(code.data()), code_size); u32 const_buffers_count{}; file.ReadBytes(&const_buffers_count, sizeof(u32)); @@ -321,12 +364,20 @@ void ShaderDiskCacheOpenGL::SaveDecompiled(u64 unique_identifier, const std::str return; } + const std::vector compressed_code{CompressData(code.data(), code.size())}; + if (compressed_code.empty()) { + LOG_ERROR(Render_OpenGL, "Failed to compress GLSL code - skipping shader {:016x}", + unique_identifier); + return; + } + file.WriteObject(static_cast(PrecompiledEntryKind::Decompiled)); file.WriteObject(unique_identifier); file.WriteObject(static_cast(code.size())); - file.WriteArray(code.data(), code.size()); + file.WriteObject(static_cast(compressed_code.size())); + file.WriteArray(compressed_code.data(), compressed_code.size()); file.WriteObject(static_cast(entries.const_buffers.size())); for (const auto& cbuf : entries.const_buffers) { From ed956569a4a56a20dc3a26d16f8920a6c63fda09 Mon Sep 17 00:00:00 2001 From: ReinUsesLisp Date: Tue, 15 Jan 2019 02:05:30 -0300 Subject: [PATCH 18/32] gl_shader_disk_cache: Compress program binaries using LZ4 --- .../renderer_opengl/gl_shader_disk_cache.cpp | 35 +++++++++++++++---- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp index bea4b29d1..f6d950b0b 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp @@ -282,9 +282,22 @@ bool ShaderDiskCacheOpenGL::LoadPrecompiled( file.ReadBytes(&dump.binary_format, sizeof(u32)); u32 binary_length{}; + u32 compressed_size{}; file.ReadBytes(&binary_length, sizeof(u32)); - dump.binary.resize(binary_length); - file.ReadBytes(dump.binary.data(), dump.binary.size()); + file.ReadBytes(&compressed_size, sizeof(u32)); + + std::vector compressed_binary(compressed_size); + file.ReadArray(compressed_binary.data(), compressed_binary.size()); + + dump.binary = DecompressData(compressed_binary, binary_length); + if (dump.binary.empty()) { + LOG_ERROR(Render_OpenGL, + "Failed to decompress precompiled binary program - removing"); + InvalidatePrecompiled(); + dumps.clear(); + decompiled.clear(); + return false; + } dumps.insert({usage, dump}); break; @@ -418,10 +431,6 @@ void ShaderDiskCacheOpenGL::SaveDump(const ShaderDiskCacheUsage& usage, GLuint p return; } - file.WriteObject(static_cast(PrecompiledEntryKind::Dump)); - - file.WriteObject(usage); - GLint binary_length{}; glGetProgramiv(program, GL_PROGRAM_BINARY_LENGTH, &binary_length); @@ -429,9 +438,21 @@ void ShaderDiskCacheOpenGL::SaveDump(const ShaderDiskCacheUsage& usage, GLuint p std::vector binary(binary_length); glGetProgramBinary(program, binary_length, nullptr, &binary_format, binary.data()); + const std::vector compressed_binary = CompressData(binary.data(), binary.size()); + if (compressed_binary.empty()) { + LOG_ERROR(Render_OpenGL, "Failed to compress binary program in shader={:016x}", + usage.unique_identifier); + return; + } + + file.WriteObject(static_cast(PrecompiledEntryKind::Dump)); + + file.WriteObject(usage); + file.WriteObject(static_cast(binary_format)); file.WriteObject(static_cast(binary_length)); - file.WriteArray(binary.data(), binary.size()); + file.WriteObject(static_cast(compressed_binary.size())); + file.WriteArray(compressed_binary.data(), compressed_binary.size()); } FileUtil::IOFile ShaderDiskCacheOpenGL::AppendTransferableFile() const { From 8ee3666a3c19c3522fc980c5bed8a519e99e0d95 Mon Sep 17 00:00:00 2001 From: ReinUsesLisp Date: Tue, 15 Jan 2019 02:17:38 -0300 Subject: [PATCH 19/32] gl_shader_disk_cache: Pass return values returning instead of by parameters --- .../renderer_opengl/gl_shader_cache.cpp | 12 ++--- .../renderer_opengl/gl_shader_disk_cache.cpp | 47 +++++++++---------- .../renderer_opengl/gl_shader_disk_cache.h | 15 +++--- 3 files changed, 36 insertions(+), 38 deletions(-) diff --git a/src/video_core/renderer_opengl/gl_shader_cache.cpp b/src/video_core/renderer_opengl/gl_shader_cache.cpp index 761b355e4..49b872c44 100644 --- a/src/video_core/renderer_opengl/gl_shader_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_cache.cpp @@ -344,17 +344,15 @@ ShaderDiskCacheUsage CachedShader::GetUsage(GLenum primitive_mode, ShaderCacheOpenGL::ShaderCacheOpenGL(RasterizerOpenGL& rasterizer) : RasterizerCache{rasterizer} {} void ShaderCacheOpenGL::LoadDiskCache() { - std::vector raws; - std::vector usages; - if (!disk_cache.LoadTransferable(raws, usages)) { + const auto transferable = disk_cache.LoadTransferable(); + if (!transferable) { return; } + const auto [raws, usages] = *transferable; - std::map decompiled; - std::map dumps; - disk_cache.LoadPrecompiled(decompiled, dumps); + auto [decompiled, dumps] = disk_cache.LoadPrecompiled(); - const std::set supported_formats{GetSupportedFormats()}; + const auto supported_formats{GetSupportedFormats()}; const auto unspecialized{GenerateUnspecializedShaders(raws, decompiled)}; // Build shaders diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp index f6d950b0b..7783e3c01 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp @@ -111,17 +111,17 @@ void ShaderDiskCacheRaw::Save(FileUtil::IOFile& file) const { } } -bool ShaderDiskCacheOpenGL::LoadTransferable(std::vector& raws, - std::vector& usages) { +std::optional, std::vector>> +ShaderDiskCacheOpenGL::LoadTransferable() { if (!Settings::values.use_disk_shader_cache) { - return false; + return {}; } FileUtil::IOFile file(GetTransferablePath(), "rb"); if (!file.IsOpen()) { LOG_INFO(Render_OpenGL, "No transferable shader cache found for game with title id={}", GetTitleID()); - return false; + return {}; } const u64 file_size = file.GetSize(); @@ -132,15 +132,17 @@ bool ShaderDiskCacheOpenGL::LoadTransferable(std::vector& ra LOG_INFO(Render_OpenGL, "Transferable shader cache is old - removing"); file.Close(); FileUtil::Delete(GetTransferablePath()); - return false; + return {}; } if (version > NativeVersion) { LOG_WARNING(Render_OpenGL, "Transferable shader cache was generated with a newer version " "of the emulator - skipping"); - return false; + return {}; } // Version is valid, load the shaders + std::vector raws; + std::vector usages; while (file.Tell() < file_size) { TransferableEntryKind kind{}; file.ReadBytes(&kind, sizeof(u32)); @@ -161,25 +163,24 @@ bool ShaderDiskCacheOpenGL::LoadTransferable(std::vector& ra default: LOG_ERROR(Render_OpenGL, "Unknown transferable shader cache entry kind={} - aborting", static_cast(kind)); - return false; + return {}; } } - return true; + return {{raws, usages}}; } -bool ShaderDiskCacheOpenGL::LoadPrecompiled( - std::map& decompiled, - std::map& dumps) { - +std::pair, + std::map> +ShaderDiskCacheOpenGL::LoadPrecompiled() { if (!Settings::values.use_disk_shader_cache) { - return false; + return {}; } FileUtil::IOFile file(GetPrecompiledPath(), "rb"); if (!file.IsOpen()) { LOG_INFO(Render_OpenGL, "No precompiled shader cache found for game with title id={}", GetTitleID()); - return false; + return {}; } const u64 file_size = file.GetSize(); @@ -189,9 +190,11 @@ bool ShaderDiskCacheOpenGL::LoadPrecompiled( LOG_INFO(Render_OpenGL, "Precompiled cache is from another version of yuzu - removing"); file.Close(); InvalidatePrecompiled(); - return false; + return {}; } + std::map decompiled; + std::map dumps; while (file.Tell() < file_size) { PrecompiledEntryKind kind{}; file.ReadBytes(&kind, sizeof(u32)); @@ -217,9 +220,7 @@ bool ShaderDiskCacheOpenGL::LoadPrecompiled( "Failed to decompress GLSL code in precompiled shader={:016x} - removing", unique_identifier); InvalidatePrecompiled(); - dumps.clear(); - decompiled.clear(); - return false; + return {}; } entry.code = std::string(reinterpret_cast(code.data()), code_size); @@ -294,9 +295,7 @@ bool ShaderDiskCacheOpenGL::LoadPrecompiled( LOG_ERROR(Render_OpenGL, "Failed to decompress precompiled binary program - removing"); InvalidatePrecompiled(); - dumps.clear(); - decompiled.clear(); - return false; + return {}; } dumps.insert({usage, dump}); @@ -306,12 +305,10 @@ bool ShaderDiskCacheOpenGL::LoadPrecompiled( LOG_ERROR(Render_OpenGL, "Unknown precompiled shader cache entry kind={} - removing", static_cast(kind)); InvalidatePrecompiled(); - dumps.clear(); - decompiled.clear(); - return false; + return {}; } } - return true; + return {decompiled, dumps}; } void ShaderDiskCacheOpenGL::InvalidateTransferable() const { diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.h b/src/video_core/renderer_opengl/gl_shader_disk_cache.h index f11693789..c44a776d9 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.h +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.h @@ -4,9 +4,11 @@ #pragma once +#include #include #include #include +#include #include #include @@ -142,13 +144,14 @@ struct ShaderDiskCacheDump { class ShaderDiskCacheOpenGL { public: - /// Loads transferable cache. If file has a old version, it deletes it. Returns true on success. - bool LoadTransferable(std::vector& raws, - std::vector& usages); + /// Loads transferable cache. If file has a old version or on failure, it deletes the file. + std::optional, std::vector>> + LoadTransferable(); - /// Loads current game's precompiled cache. Invalidates if emulator's version has changed. - bool LoadPrecompiled(std::map& decompiled, - std::map& dumps); + /// Loads current game's precompiled cache. Invalidates on failure. + std::pair, + std::map> + LoadPrecompiled(); /// Removes the transferable (and precompiled) cache file. void InvalidateTransferable() const; From 750abcc23d0b9584b716ab93110383209b0971f8 Mon Sep 17 00:00:00 2001 From: ReinUsesLisp Date: Tue, 15 Jan 2019 02:42:25 -0300 Subject: [PATCH 20/32] gl_shader_disk_cache: Address miscellaneous feedback --- .../renderer_opengl/gl_shader_cache.cpp | 6 +-- .../renderer_opengl/gl_shader_cache.h | 6 +-- .../renderer_opengl/gl_shader_decompiler.h | 8 +-- .../renderer_opengl/gl_shader_disk_cache.cpp | 26 ++++++--- .../renderer_opengl/gl_shader_disk_cache.h | 54 ++++++++++--------- 5 files changed, 57 insertions(+), 43 deletions(-) diff --git a/src/video_core/renderer_opengl/gl_shader_cache.cpp b/src/video_core/renderer_opengl/gl_shader_cache.cpp index 49b872c44..6acfd1649 100644 --- a/src/video_core/renderer_opengl/gl_shader_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_cache.cpp @@ -181,7 +181,8 @@ CachedProgram SpecializeShader(const std::string& code, const GLShader::ShaderEn } if (program_type == Maxwell::ShaderProgram::Geometry) { - const auto [glsl_topology, _, max_vertices] = GetPrimitiveDescription(primitive_mode); + const auto [glsl_topology, debug_name, max_vertices] = + GetPrimitiveDescription(primitive_mode); source += "layout (" + std::string(glsl_topology) + ") in;\n"; source += "#define MAX_VERTEX_INPUT " + std::to_string(max_vertices) + '\n'; @@ -314,7 +315,7 @@ GLuint CachedShader::LazyGeometryProgram(CachedProgram& target_program, BaseBind if (target_program) { return target_program->handle; } - const auto [_, debug_name, __] = GetPrimitiveDescription(primitive_mode); + const auto [glsl_name, debug_name, vertices] = GetPrimitiveDescription(primitive_mode); target_program = TryLoadProgram(primitive_mode, base_bindings); if (!target_program) { target_program = @@ -419,7 +420,6 @@ CachedProgram ShaderCacheOpenGL::GeneratePrecompiledProgram( std::map ShaderCacheOpenGL::GenerateUnspecializedShaders( const std::vector& raws, const std::map& decompiled) { - std::map unspecialized; for (const auto& raw : raws) { diff --git a/src/video_core/renderer_opengl/gl_shader_cache.h b/src/video_core/renderer_opengl/gl_shader_cache.h index 3b5a82f8a..c6a621ae3 100644 --- a/src/video_core/renderer_opengl/gl_shader_cache.h +++ b/src/video_core/renderer_opengl/gl_shader_cache.h @@ -86,9 +86,9 @@ private: ShaderDiskCacheUsage GetUsage(GLenum primitive_mode, BaseBindings base_bindings) const; - const VAddr addr; - const u64 unique_identifier; - const Maxwell::ShaderProgram program_type; + VAddr addr{}; + u64 unique_identifier{}; + Maxwell::ShaderProgram program_type{}; ShaderDiskCacheOpenGL& disk_cache; const PrecompiledPrograms& precompiled_programs; diff --git a/src/video_core/renderer_opengl/gl_shader_decompiler.h b/src/video_core/renderer_opengl/gl_shader_decompiler.h index 0031cb614..72aca4938 100644 --- a/src/video_core/renderer_opengl/gl_shader_decompiler.h +++ b/src/video_core/renderer_opengl/gl_shader_decompiler.h @@ -19,7 +19,11 @@ class ShaderIR; namespace OpenGL::GLShader { +struct ShaderEntries; + using Maxwell = Tegra::Engines::Maxwell3D::Regs; +using ProgramResult = std::pair; +using SamplerEntry = VideoCommon::Shader::Sampler; class ConstBufferEntry : public VideoCommon::Shader::ConstBuffer { public: @@ -34,8 +38,6 @@ private: u32 index{}; }; -using SamplerEntry = VideoCommon::Shader::Sampler; - class GlobalMemoryEntry { public: explicit GlobalMemoryEntry(u32 cbuf_index, u32 cbuf_offset) @@ -62,8 +64,6 @@ struct ShaderEntries { std::size_t shader_length{}; }; -using ProgramResult = std::pair; - std::string GetCommonDeclarations(); ProgramResult Decompile(const VideoCommon::Shader::ShaderIR& ir, Maxwell::ShaderStage stage, diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp index 7783e3c01..5157e319a 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp @@ -49,7 +49,7 @@ std::string GetTitleID() { std::string GetShaderHash() { std::array hash{}; std::strncpy(hash.data(), Common::g_shader_cache_version, ShaderHashSize); - return std::string(hash.data()); + return std::string(hash.data(), hash.size()); } template @@ -86,7 +86,8 @@ ShaderDiskCacheRaw::ShaderDiskCacheRaw(FileUtil::IOFile& file) { file.ReadBytes(&unique_identifier, sizeof(u64)); file.ReadBytes(&program_type, sizeof(u32)); - u32 program_code_size{}, program_code_size_b{}; + u32 program_code_size{}; + u32 program_code_size_b{}; file.ReadBytes(&program_code_size, sizeof(u32)); file.ReadBytes(&program_code_size_b, sizeof(u32)); @@ -99,6 +100,15 @@ ShaderDiskCacheRaw::ShaderDiskCacheRaw(FileUtil::IOFile& file) { } } +ShaderDiskCacheRaw::ShaderDiskCacheRaw(u64 unique_identifier, Maxwell::ShaderProgram program_type, + u32 program_code_size, u32 program_code_size_b, + ProgramCode program_code, ProgramCode program_code_b) + : unique_identifier{unique_identifier}, program_type{program_type}, + program_code_size{program_code_size}, program_code_size_b{program_code_size_b}, + program_code{std::move(program_code)}, program_code_b{std::move(program_code_b)} {} + +ShaderDiskCacheRaw::~ShaderDiskCacheRaw() = default; + void ShaderDiskCacheRaw::Save(FileUtil::IOFile& file) const { file.WriteObject(unique_identifier); file.WriteObject(static_cast(program_type)); @@ -186,7 +196,7 @@ ShaderDiskCacheOpenGL::LoadPrecompiled() { char precompiled_hash[ShaderHashSize]; file.ReadBytes(&precompiled_hash, ShaderHashSize); - if (std::string(precompiled_hash) != GetShaderHash()) { + if (precompiled_hash != GetShaderHash()) { LOG_INFO(Render_OpenGL, "Precompiled cache is from another version of yuzu - removing"); file.Close(); InvalidatePrecompiled(); @@ -311,13 +321,13 @@ ShaderDiskCacheOpenGL::LoadPrecompiled() { return {decompiled, dumps}; } -void ShaderDiskCacheOpenGL::InvalidateTransferable() const { - FileUtil::Delete(GetTransferablePath()); - InvalidatePrecompiled(); +bool ShaderDiskCacheOpenGL::InvalidateTransferable() const { + const bool success = FileUtil::Delete(GetTransferablePath()); + return InvalidatePrecompiled() && success; } -void ShaderDiskCacheOpenGL::InvalidatePrecompiled() const { - FileUtil::Delete(GetPrecompiledPath()); +bool ShaderDiskCacheOpenGL::InvalidatePrecompiled() const { + return FileUtil::Delete(GetPrecompiledPath()); } void ShaderDiskCacheOpenGL::SaveRaw(const ShaderDiskCacheRaw& entry) { diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.h b/src/video_core/renderer_opengl/gl_shader_disk_cache.h index c44a776d9..4bffe4307 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.h +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.h @@ -15,22 +15,20 @@ #include "common/assert.h" #include "common/common_types.h" -#include "common/file_util.h" #include "video_core/engines/maxwell_3d.h" #include "video_core/renderer_opengl/gl_shader_gen.h" +namespace FileUtil { +class IOFile; +} // namespace FileUtil + namespace OpenGL { using ProgramCode = std::vector; using Maxwell = Tegra::Engines::Maxwell3D::Regs; +/// Allocated bindings used by an OpenGL shader program struct BaseBindings { -private: - auto Tie() const { - return std::tie(cbuf, gmem, sampler); - } - -public: u32 cbuf{}; u32 gmem{}; u32 sampler{}; @@ -44,20 +42,24 @@ public: } bool operator!=(const BaseBindings& rhs) const { - return !this->operator==(rhs); + return !operator==(rhs); + } + + std::tuple Tie() const { + return std::tie(cbuf, gmem, sampler); } }; +/// Describes a shader how it's used by the guest GPU class ShaderDiskCacheRaw { public: explicit ShaderDiskCacheRaw(FileUtil::IOFile& file); explicit ShaderDiskCacheRaw(u64 unique_identifier, Maxwell::ShaderProgram program_type, u32 program_code_size, u32 program_code_size_b, - ProgramCode program_code, ProgramCode program_code_b) - : unique_identifier{unique_identifier}, program_type{program_type}, - program_code_size{program_code_size}, program_code_size_b{program_code_size_b}, - program_code{std::move(program_code)}, program_code_b{std::move(program_code_b)} {} + ProgramCode program_code, ProgramCode program_code_b); + + ~ShaderDiskCacheRaw(); void Save(FileUtil::IOFile& file) const; @@ -108,17 +110,8 @@ private: ProgramCode program_code_b; }; +/// Describes how a shader is used struct ShaderDiskCacheUsage { -private: - auto Tie() const { - return std::tie(unique_identifier, bindings, primitive); - } - -public: - u64 unique_identifier{}; - BaseBindings bindings; - GLenum primitive{}; - bool operator<(const ShaderDiskCacheUsage& rhs) const { return Tie() < rhs.Tie(); } @@ -128,15 +121,26 @@ public: } bool operator!=(const ShaderDiskCacheUsage& rhs) const { - return !this->operator==(rhs); + return !operator==(rhs); + } + + u64 unique_identifier{}; + BaseBindings bindings; + GLenum primitive{}; + +private: + std::tuple Tie() const { + return std::tie(unique_identifier, bindings, primitive); } }; +/// Contains decompiled data from a shader struct ShaderDiskCacheDecompiled { std::string code; GLShader::ShaderEntries entries; }; +/// Contains an OpenGL dumped binary program struct ShaderDiskCacheDump { GLenum binary_format; std::vector binary; @@ -154,10 +158,10 @@ public: LoadPrecompiled(); /// Removes the transferable (and precompiled) cache file. - void InvalidateTransferable() const; + bool InvalidateTransferable() const; /// Removes the precompiled cache file. - void InvalidatePrecompiled() const; + bool InvalidatePrecompiled() const; /// Saves a raw dump to the transferable file. Checks for collisions. void SaveRaw(const ShaderDiskCacheRaw& entry); From 2bc6a699dc53baf55b0ccbb40750a40036ee184f Mon Sep 17 00:00:00 2001 From: ReinUsesLisp Date: Tue, 15 Jan 2019 15:22:25 -0300 Subject: [PATCH 21/32] gl_shader_disk_cache: Guard reads and writes against failure --- .../renderer_opengl/gl_shader_disk_cache.cpp | 531 +++++++++++------- .../renderer_opengl/gl_shader_disk_cache.h | 30 +- 2 files changed, 342 insertions(+), 219 deletions(-) diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp index 5157e319a..f8bdb7779 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp @@ -24,6 +24,8 @@ namespace OpenGL { +using ShaderCacheVersionHash = std::array; + enum class TransferableEntryKind : u32 { Raw, Usage, @@ -35,7 +37,6 @@ enum class PrecompiledEntryKind : u32 { }; constexpr u32 NativeVersion = 1; -constexpr u32 ShaderHashSize = 64; // Making sure sizes doesn't change by accident static_assert(sizeof(BaseBindings) == 12); @@ -46,10 +47,11 @@ std::string GetTitleID() { return fmt::format("{:016X}", Core::CurrentProcess()->GetTitleID()); } -std::string GetShaderHash() { - std::array hash{}; - std::strncpy(hash.data(), Common::g_shader_cache_version, ShaderHashSize); - return std::string(hash.data(), hash.size()); +ShaderCacheVersionHash GetShaderCacheVersionHash() { + ShaderCacheVersionHash hash{}; + const std::size_t length = std::min(std::strlen(Common::g_shader_cache_version), hash.size()); + std::memcpy(hash.data(), Common::g_shader_cache_version, length); + return hash; } template @@ -82,24 +84,6 @@ std::vector DecompressData(const std::vector& compressed, std::size_t un } } // namespace -ShaderDiskCacheRaw::ShaderDiskCacheRaw(FileUtil::IOFile& file) { - file.ReadBytes(&unique_identifier, sizeof(u64)); - file.ReadBytes(&program_type, sizeof(u32)); - - u32 program_code_size{}; - u32 program_code_size_b{}; - file.ReadBytes(&program_code_size, sizeof(u32)); - file.ReadBytes(&program_code_size_b, sizeof(u32)); - - program_code.resize(program_code_size); - program_code_b.resize(program_code_size_b); - - file.ReadArray(program_code.data(), program_code_size); - if (HasProgramA()) { - file.ReadArray(program_code_b.data(), program_code_size_b); - } -} - ShaderDiskCacheRaw::ShaderDiskCacheRaw(u64 unique_identifier, Maxwell::ShaderProgram program_type, u32 program_code_size, u32 program_code_size_b, ProgramCode program_code, ProgramCode program_code_b) @@ -107,25 +91,57 @@ ShaderDiskCacheRaw::ShaderDiskCacheRaw(u64 unique_identifier, Maxwell::ShaderPro program_code_size{program_code_size}, program_code_size_b{program_code_size_b}, program_code{std::move(program_code)}, program_code_b{std::move(program_code_b)} {} +ShaderDiskCacheRaw::ShaderDiskCacheRaw() = default; + ShaderDiskCacheRaw::~ShaderDiskCacheRaw() = default; -void ShaderDiskCacheRaw::Save(FileUtil::IOFile& file) const { - file.WriteObject(unique_identifier); - file.WriteObject(static_cast(program_type)); - file.WriteObject(program_code_size); - file.WriteObject(program_code_size_b); - - file.WriteArray(program_code.data(), program_code_size); - if (HasProgramA()) { - file.WriteArray(program_code_b.data(), program_code_size_b); +bool ShaderDiskCacheRaw::Load(FileUtil::IOFile& file) { + if (file.ReadBytes(&unique_identifier, sizeof(u64)) != sizeof(u64) || + file.ReadBytes(&program_type, sizeof(u32)) != sizeof(u32)) { + return false; } + u32 program_code_size{}; + u32 program_code_size_b{}; + if (file.ReadBytes(&program_code_size, sizeof(u32)) != sizeof(u32) || + file.ReadBytes(&program_code_size_b, sizeof(u32)) != sizeof(u32)) { + return false; + } + + program_code.resize(program_code_size); + program_code_b.resize(program_code_size_b); + + if (file.ReadArray(program_code.data(), program_code_size) != program_code_size) + return false; + + if (HasProgramA() && + file.ReadArray(program_code_b.data(), program_code_size_b) != program_code_size_b) { + return false; + } + return true; +} + +bool ShaderDiskCacheRaw::Save(FileUtil::IOFile& file) const { + if (file.WriteObject(unique_identifier) != 1 || + file.WriteObject(static_cast(program_type)) != 1 || + file.WriteObject(program_code_size) != 1 || file.WriteObject(program_code_size_b) != 1) { + return false; + } + + if (file.WriteArray(program_code.data(), program_code_size) != program_code_size) + return false; + + if (HasProgramA() && + file.WriteArray(program_code_b.data(), program_code_size_b) != program_code_size_b) { + return false; + } + return true; } std::optional, std::vector>> ShaderDiskCacheOpenGL::LoadTransferable() { - if (!Settings::values.use_disk_shader_cache) { + if (!Settings::values.use_disk_shader_cache) return {}; - } + tried_to_load = true; FileUtil::IOFile file(GetTransferablePath(), "rb"); if (!file.IsOpen()) { @@ -133,15 +149,19 @@ ShaderDiskCacheOpenGL::LoadTransferable() { GetTitleID()); return {}; } - const u64 file_size = file.GetSize(); u32 version{}; - file.ReadBytes(&version, sizeof(version)); + if (file.ReadBytes(&version, sizeof(version)) != sizeof(version)) { + LOG_ERROR(Render_OpenGL, + "Failed to get transferable cache version for title id={} - skipping", + GetTitleID()); + return {}; + } if (version < NativeVersion) { LOG_INFO(Render_OpenGL, "Transferable shader cache is old - removing"); file.Close(); - FileUtil::Delete(GetTransferablePath()); + InvalidateTransferable(); return {}; } if (version > NativeVersion) { @@ -153,25 +173,35 @@ ShaderDiskCacheOpenGL::LoadTransferable() { // Version is valid, load the shaders std::vector raws; std::vector usages; - while (file.Tell() < file_size) { + while (file.Tell() < file.GetSize()) { TransferableEntryKind kind{}; - file.ReadBytes(&kind, sizeof(u32)); + if (file.ReadBytes(&kind, sizeof(u32)) != sizeof(u32)) { + LOG_ERROR(Render_OpenGL, "Failed to read transferable file - skipping"); + return {}; + } switch (kind) { case TransferableEntryKind::Raw: { - ShaderDiskCacheRaw entry{file}; + ShaderDiskCacheRaw entry; + if (!entry.Load(file)) { + LOG_ERROR(Render_OpenGL, "Failed to load transferable raw entry - skipping"); + return {}; + } transferable.insert({entry.GetUniqueIdentifier(), {}}); raws.push_back(std::move(entry)); break; } case TransferableEntryKind::Usage: { ShaderDiskCacheUsage usage{}; - file.ReadBytes(&usage, sizeof(usage)); + if (file.ReadBytes(&usage, sizeof(usage)) != sizeof(usage)) { + LOG_ERROR(Render_OpenGL, "Failed to load transferable usage entry - skipping"); + return {}; + } usages.push_back(std::move(usage)); break; } default: - LOG_ERROR(Render_OpenGL, "Unknown transferable shader cache entry kind={} - aborting", + LOG_ERROR(Render_OpenGL, "Unknown transferable shader cache entry kind={} - skipping", static_cast(kind)); return {}; } @@ -182,9 +212,8 @@ ShaderDiskCacheOpenGL::LoadTransferable() { std::pair, std::map> ShaderDiskCacheOpenGL::LoadPrecompiled() { - if (!Settings::values.use_disk_shader_cache) { + if (!IsUsable()) return {}; - } FileUtil::IOFile file(GetPrecompiledPath(), "rb"); if (!file.IsOpen()) { @@ -192,119 +221,75 @@ ShaderDiskCacheOpenGL::LoadPrecompiled() { GetTitleID()); return {}; } - const u64 file_size = file.GetSize(); - char precompiled_hash[ShaderHashSize]; - file.ReadBytes(&precompiled_hash, ShaderHashSize); - if (precompiled_hash != GetShaderHash()) { - LOG_INFO(Render_OpenGL, "Precompiled cache is from another version of yuzu - removing"); + const auto result = LoadPrecompiledFile(file); + if (!result) { + LOG_INFO(Render_OpenGL, + "Failed to load precompiled cache for game with title id={} - removing", + GetTitleID()); file.Close(); InvalidatePrecompiled(); return {}; } + return *result; +} + +std::optional, + std::map>> +ShaderDiskCacheOpenGL::LoadPrecompiledFile(FileUtil::IOFile& file) { + ShaderCacheVersionHash file_hash{}; + if (file.ReadArray(file_hash.data(), file_hash.size()) != file_hash.size()) { + return {}; + } + if (GetShaderCacheVersionHash() != file_hash) { + LOG_INFO(Render_OpenGL, "Precompiled cache is from another version of the emulator"); + return {}; + } std::map decompiled; std::map dumps; - while (file.Tell() < file_size) { + while (file.Tell() < file.GetSize()) { PrecompiledEntryKind kind{}; - file.ReadBytes(&kind, sizeof(u32)); + if (file.ReadBytes(&kind, sizeof(u32)) != sizeof(u32)) { + return {}; + } switch (kind) { case PrecompiledEntryKind::Decompiled: { - ShaderDiskCacheDecompiled entry; - u64 unique_identifier{}; - file.ReadBytes(&unique_identifier, sizeof(u64)); - - u32 code_size{}; - u32 compressed_code_size{}; - file.ReadBytes(&code_size, sizeof(u32)); - file.ReadBytes(&compressed_code_size, sizeof(u32)); - - std::vector compressed_code(compressed_code_size); - file.ReadArray(compressed_code.data(), compressed_code.size()); - - const std::vector code = DecompressData(compressed_code, code_size); - if (code.empty()) { - LOG_ERROR(Render_OpenGL, - "Failed to decompress GLSL code in precompiled shader={:016x} - removing", - unique_identifier); - InvalidatePrecompiled(); + if (file.ReadBytes(&unique_identifier, sizeof(u64)) != sizeof(u64)) return {}; - } - entry.code = std::string(reinterpret_cast(code.data()), code_size); - u32 const_buffers_count{}; - file.ReadBytes(&const_buffers_count, sizeof(u32)); - for (u32 i = 0; i < const_buffers_count; ++i) { - u32 max_offset{}, index{}; - u8 is_indirect{}; - file.ReadBytes(&max_offset, sizeof(u32)); - file.ReadBytes(&index, sizeof(u32)); - file.ReadBytes(&is_indirect, sizeof(u8)); - - entry.entries.const_buffers.emplace_back(max_offset, is_indirect != 0, index); - } - - u32 samplers_count{}; - file.ReadBytes(&samplers_count, sizeof(u32)); - for (u32 i = 0; i < samplers_count; ++i) { - u64 offset{}, index{}; - u32 type{}; - u8 is_array{}, is_shadow{}; - file.ReadBytes(&offset, sizeof(u64)); - file.ReadBytes(&index, sizeof(u64)); - file.ReadBytes(&type, sizeof(u32)); - file.ReadBytes(&is_array, sizeof(u8)); - file.ReadBytes(&is_shadow, sizeof(u8)); - - entry.entries.samplers.emplace_back( - static_cast(offset), static_cast(index), - static_cast(type), is_array != 0, is_shadow != 0); - } - - u32 global_memory_count{}; - file.ReadBytes(&global_memory_count, sizeof(u32)); - for (u32 i = 0; i < global_memory_count; ++i) { - u32 cbuf_index{}, cbuf_offset{}; - file.ReadBytes(&cbuf_index, sizeof(u32)); - file.ReadBytes(&cbuf_offset, sizeof(u32)); - entry.entries.global_memory_entries.emplace_back(cbuf_index, cbuf_offset); - } - - for (auto& clip_distance : entry.entries.clip_distances) { - u8 clip_distance_raw{}; - file.ReadBytes(&clip_distance_raw, sizeof(u8)); - clip_distance = clip_distance_raw != 0; - } - - u64 shader_length{}; - file.ReadBytes(&shader_length, sizeof(u64)); - entry.entries.shader_length = static_cast(shader_length); - - decompiled.insert({unique_identifier, std::move(entry)}); + const auto entry = LoadDecompiledEntry(file); + if (!entry) + return {}; + decompiled.insert({unique_identifier, std::move(*entry)}); break; } case PrecompiledEntryKind::Dump: { ShaderDiskCacheUsage usage; - file.ReadBytes(&usage, sizeof(usage)); + if (file.ReadBytes(&usage, sizeof(usage)) != sizeof(usage)) + return {}; ShaderDiskCacheDump dump; - file.ReadBytes(&dump.binary_format, sizeof(u32)); + if (file.ReadBytes(&dump.binary_format, sizeof(u32)) != sizeof(u32)) + return {}; u32 binary_length{}; u32 compressed_size{}; - file.ReadBytes(&binary_length, sizeof(u32)); - file.ReadBytes(&compressed_size, sizeof(u32)); + if (file.ReadBytes(&binary_length, sizeof(u32)) != sizeof(u32) || + file.ReadBytes(&compressed_size, sizeof(u32)) != sizeof(u32)) { + return {}; + } std::vector compressed_binary(compressed_size); - file.ReadArray(compressed_binary.data(), compressed_binary.size()); + if (file.ReadArray(compressed_binary.data(), compressed_binary.size()) != + compressed_binary.size()) { + return {}; + } dump.binary = DecompressData(compressed_binary, binary_length); if (dump.binary.empty()) { - LOG_ERROR(Render_OpenGL, - "Failed to decompress precompiled binary program - removing"); - InvalidatePrecompiled(); return {}; } @@ -312,28 +297,165 @@ ShaderDiskCacheOpenGL::LoadPrecompiled() { break; } default: - LOG_ERROR(Render_OpenGL, "Unknown precompiled shader cache entry kind={} - removing", - static_cast(kind)); - InvalidatePrecompiled(); return {}; } } - return {decompiled, dumps}; + return {{decompiled, dumps}}; } -bool ShaderDiskCacheOpenGL::InvalidateTransferable() const { - const bool success = FileUtil::Delete(GetTransferablePath()); - return InvalidatePrecompiled() && success; +std::optional ShaderDiskCacheOpenGL::LoadDecompiledEntry( + FileUtil::IOFile& file) { + u32 code_size{}; + u32 compressed_code_size{}; + if (file.ReadBytes(&code_size, sizeof(u32)) != sizeof(u32) || + file.ReadBytes(&compressed_code_size, sizeof(u32)) != sizeof(u32)) { + return {}; + } + + std::vector compressed_code(compressed_code_size); + if (file.ReadArray(compressed_code.data(), compressed_code.size()) != compressed_code.size()) { + return {}; + } + + const std::vector code = DecompressData(compressed_code, code_size); + if (code.empty()) { + return {}; + } + ShaderDiskCacheDecompiled entry; + entry.code = std::string(reinterpret_cast(code.data()), code_size); + + u32 const_buffers_count{}; + if (file.ReadBytes(&const_buffers_count, sizeof(u32)) != sizeof(u32)) + return {}; + for (u32 i = 0; i < const_buffers_count; ++i) { + u32 max_offset{}; + u32 index{}; + u8 is_indirect{}; + if (file.ReadBytes(&max_offset, sizeof(u32)) != sizeof(u32) || + file.ReadBytes(&index, sizeof(u32)) != sizeof(u32) || + file.ReadBytes(&is_indirect, sizeof(u8)) != sizeof(u8)) { + return {}; + } + entry.entries.const_buffers.emplace_back(max_offset, is_indirect != 0, index); + } + + u32 samplers_count{}; + if (file.ReadBytes(&samplers_count, sizeof(u32)) != sizeof(u32)) + return {}; + for (u32 i = 0; i < samplers_count; ++i) { + u64 offset{}; + u64 index{}; + u32 type{}; + u8 is_array{}; + u8 is_shadow{}; + if (file.ReadBytes(&offset, sizeof(u64)) != sizeof(u64) || + file.ReadBytes(&index, sizeof(u64)) != sizeof(u64) || + file.ReadBytes(&type, sizeof(u32)) != sizeof(u32) || + file.ReadBytes(&is_array, sizeof(u8)) != sizeof(u8) || + file.ReadBytes(&is_shadow, sizeof(u8)) != sizeof(u8)) { + return {}; + } + entry.entries.samplers.emplace_back( + static_cast(offset), static_cast(index), + static_cast(type), is_array != 0, is_shadow != 0); + } + + u32 global_memory_count{}; + if (file.ReadBytes(&global_memory_count, sizeof(u32)) != sizeof(u32)) + return {}; + for (u32 i = 0; i < global_memory_count; ++i) { + u32 cbuf_index{}; + u32 cbuf_offset{}; + if (file.ReadBytes(&cbuf_index, sizeof(u32)) != sizeof(u32) || + file.ReadBytes(&cbuf_offset, sizeof(u32)) != sizeof(u32)) { + return {}; + } + entry.entries.global_memory_entries.emplace_back(cbuf_index, cbuf_offset); + } + + for (auto& clip_distance : entry.entries.clip_distances) { + u8 clip_distance_raw{}; + if (file.ReadBytes(&clip_distance_raw, sizeof(u8)) != sizeof(u8)) + return {}; + clip_distance = clip_distance_raw != 0; + } + + u64 shader_length{}; + if (file.ReadBytes(&shader_length, sizeof(u64)) != sizeof(u64)) + return {}; + entry.entries.shader_length = static_cast(shader_length); + + return entry; } -bool ShaderDiskCacheOpenGL::InvalidatePrecompiled() const { - return FileUtil::Delete(GetPrecompiledPath()); +bool ShaderDiskCacheOpenGL::SaveDecompiledFile(FileUtil::IOFile& file, u64 unique_identifier, + const std::string& code, + const std::vector& compressed_code, + const GLShader::ShaderEntries& entries) { + if (file.WriteObject(static_cast(PrecompiledEntryKind::Decompiled)) != 1 || + file.WriteObject(unique_identifier) != 1 || + file.WriteObject(static_cast(code.size())) != 1 || + file.WriteObject(static_cast(compressed_code.size())) != 1 || + file.WriteArray(compressed_code.data(), compressed_code.size()) != compressed_code.size()) { + return false; + } + + if (file.WriteObject(static_cast(entries.const_buffers.size())) != 1) + return false; + for (const auto& cbuf : entries.const_buffers) { + if (file.WriteObject(static_cast(cbuf.GetMaxOffset())) != 1 || + file.WriteObject(static_cast(cbuf.GetIndex())) != 1 || + file.WriteObject(static_cast(cbuf.IsIndirect() ? 1 : 0)) != 1) { + return false; + } + } + + if (file.WriteObject(static_cast(entries.samplers.size())) != 1) + return false; + for (const auto& sampler : entries.samplers) { + if (file.WriteObject(static_cast(sampler.GetOffset())) != 1 || + file.WriteObject(static_cast(sampler.GetIndex())) != 1 || + file.WriteObject(static_cast(sampler.GetType())) != 1 || + file.WriteObject(static_cast(sampler.IsArray() ? 1 : 0)) != 1 || + file.WriteObject(static_cast(sampler.IsShadow() ? 1 : 0)) != 1) { + return false; + } + } + + if (file.WriteObject(static_cast(entries.global_memory_entries.size())) != 1) + return false; + for (const auto& gmem : entries.global_memory_entries) { + if (file.WriteObject(static_cast(gmem.GetCbufIndex())) != 1 || + file.WriteObject(static_cast(gmem.GetCbufOffset())) != 1) { + return false; + } + } + + for (const bool clip_distance : entries.clip_distances) { + if (file.WriteObject(static_cast(clip_distance ? 1 : 0)) != 1) + return false; + } + + return file.WriteObject(static_cast(entries.shader_length)) == 1; +} + +void ShaderDiskCacheOpenGL::InvalidateTransferable() const { + if (!FileUtil::Delete(GetTransferablePath())) { + LOG_ERROR(Render_OpenGL, "Failed to invalidate transferable file={}", + GetTransferablePath()); + } + InvalidatePrecompiled(); +} + +void ShaderDiskCacheOpenGL::InvalidatePrecompiled() const { + if (!FileUtil::Delete(GetPrecompiledPath())) { + LOG_ERROR(Render_OpenGL, "Failed to invalidate precompiled file={}", GetPrecompiledPath()); + } } void ShaderDiskCacheOpenGL::SaveRaw(const ShaderDiskCacheRaw& entry) { - if (!Settings::values.use_disk_shader_cache) { + if (!IsUsable()) return; - } const u64 id = entry.GetUniqueIdentifier(); if (transferable.find(id) != transferable.end()) { @@ -342,47 +464,44 @@ void ShaderDiskCacheOpenGL::SaveRaw(const ShaderDiskCacheRaw& entry) { } FileUtil::IOFile file = AppendTransferableFile(); - if (!file.IsOpen()) { + if (!file.IsOpen()) + return; + if (file.WriteObject(TransferableEntryKind::Raw) != 1 || !entry.Save(file)) { + LOG_ERROR(Render_OpenGL, "Failed to save raw transferable cache entry - removing"); + file.Close(); + InvalidateTransferable(); return; } - file.WriteObject(TransferableEntryKind::Raw); - entry.Save(file); - transferable.insert({id, {}}); } void ShaderDiskCacheOpenGL::SaveUsage(const ShaderDiskCacheUsage& usage) { - if (!Settings::values.use_disk_shader_cache) { + if (!IsUsable()) return; - } const auto it = transferable.find(usage.unique_identifier); - if (it == transferable.end()) { - LOG_CRITICAL(Render_OpenGL, "Saving shader usage without storing raw previously"); - UNREACHABLE(); - } + ASSERT_MSG(it != transferable.end(), "Saving shader usage without storing raw previously"); + auto& usages{it->second}; ASSERT(usages.find(usage) == usages.end()); usages.insert(usage); FileUtil::IOFile file = AppendTransferableFile(); - if (!file.IsOpen()) { + if (!file.IsOpen()) + return; + + if (file.WriteObject(TransferableEntryKind::Usage) != 1 || file.WriteObject(usage) != 1) { + LOG_ERROR(Render_OpenGL, "Failed to save usage transferable cache entry - removing"); + file.Close(); + InvalidateTransferable(); return; } - file.WriteObject(TransferableEntryKind::Usage); - file.WriteObject(usage); } void ShaderDiskCacheOpenGL::SaveDecompiled(u64 unique_identifier, const std::string& code, const GLShader::ShaderEntries& entries) { - if (!Settings::values.use_disk_shader_cache) { + if (!IsUsable()) return; - } - - FileUtil::IOFile file = AppendPrecompiledFile(); - if (!file.IsOpen()) { - return; - } const std::vector compressed_code{CompressData(code.data(), code.size())}; if (compressed_code.empty()) { @@ -391,52 +510,21 @@ void ShaderDiskCacheOpenGL::SaveDecompiled(u64 unique_identifier, const std::str return; } - file.WriteObject(static_cast(PrecompiledEntryKind::Decompiled)); + FileUtil::IOFile file = AppendPrecompiledFile(); + if (!file.IsOpen()) + return; - file.WriteObject(unique_identifier); - - file.WriteObject(static_cast(code.size())); - file.WriteObject(static_cast(compressed_code.size())); - file.WriteArray(compressed_code.data(), compressed_code.size()); - - file.WriteObject(static_cast(entries.const_buffers.size())); - for (const auto& cbuf : entries.const_buffers) { - file.WriteObject(static_cast(cbuf.GetMaxOffset())); - file.WriteObject(static_cast(cbuf.GetIndex())); - file.WriteObject(static_cast(cbuf.IsIndirect() ? 1 : 0)); + if (!SaveDecompiledFile(file, unique_identifier, code, compressed_code, entries)) { + LOG_ERROR(Render_OpenGL, + "Failed to save decompiled entry to the precompiled file - removing"); + file.Close(); + InvalidatePrecompiled(); } - - file.WriteObject(static_cast(entries.samplers.size())); - for (const auto& sampler : entries.samplers) { - file.WriteObject(static_cast(sampler.GetOffset())); - file.WriteObject(static_cast(sampler.GetIndex())); - file.WriteObject(static_cast(sampler.GetType())); - file.WriteObject(static_cast(sampler.IsArray() ? 1 : 0)); - file.WriteObject(static_cast(sampler.IsShadow() ? 1 : 0)); - } - - file.WriteObject(static_cast(entries.global_memory_entries.size())); - for (const auto& gmem : entries.global_memory_entries) { - file.WriteObject(static_cast(gmem.GetCbufIndex())); - file.WriteObject(static_cast(gmem.GetCbufOffset())); - } - - for (const bool clip_distance : entries.clip_distances) { - file.WriteObject(static_cast(clip_distance ? 1 : 0)); - } - - file.WriteObject(static_cast(entries.shader_length)); } void ShaderDiskCacheOpenGL::SaveDump(const ShaderDiskCacheUsage& usage, GLuint program) { - if (!Settings::values.use_disk_shader_cache) { + if (!IsUsable()) return; - } - - FileUtil::IOFile file = AppendPrecompiledFile(); - if (!file.IsOpen()) { - return; - } GLint binary_length{}; glGetProgramiv(program, GL_PROGRAM_BINARY_LENGTH, &binary_length); @@ -452,20 +540,31 @@ void ShaderDiskCacheOpenGL::SaveDump(const ShaderDiskCacheUsage& usage, GLuint p return; } - file.WriteObject(static_cast(PrecompiledEntryKind::Dump)); + FileUtil::IOFile file = AppendPrecompiledFile(); + if (!file.IsOpen()) + return; - file.WriteObject(usage); + if (file.WriteObject(static_cast(PrecompiledEntryKind::Dump)) != 1 || + file.WriteObject(usage) != 1 || file.WriteObject(static_cast(binary_format)) != 1 || + file.WriteObject(static_cast(binary_length)) != 1 || + file.WriteObject(static_cast(compressed_binary.size())) != 1 || + file.WriteArray(compressed_binary.data(), compressed_binary.size()) != + compressed_binary.size()) { + LOG_ERROR(Render_OpenGL, "Failed to save binary program file in shader={:016x} - removing", + usage.unique_identifier); + file.Close(); + InvalidatePrecompiled(); + return; + } +} - file.WriteObject(static_cast(binary_format)); - file.WriteObject(static_cast(binary_length)); - file.WriteObject(static_cast(compressed_binary.size())); - file.WriteArray(compressed_binary.data(), compressed_binary.size()); +bool ShaderDiskCacheOpenGL::IsUsable() const { + return tried_to_load && Settings::values.use_disk_shader_cache; } FileUtil::IOFile ShaderDiskCacheOpenGL::AppendTransferableFile() const { - if (!EnsureDirectories()) { + if (!EnsureDirectories()) return {}; - } const auto transferable_path{GetTransferablePath()}; const bool existed = FileUtil::Exists(transferable_path); @@ -477,15 +576,18 @@ FileUtil::IOFile ShaderDiskCacheOpenGL::AppendTransferableFile() const { } if (!existed || file.GetSize() == 0) { // If the file didn't exist, write its version - file.WriteObject(NativeVersion); + if (file.WriteObject(NativeVersion) != 1) { + LOG_ERROR(Render_OpenGL, "Failed to write transferable cache version in path={}", + transferable_path); + return {}; + } } return file; } FileUtil::IOFile ShaderDiskCacheOpenGL::AppendPrecompiledFile() const { - if (!EnsureDirectories()) { + if (!EnsureDirectories()) return {}; - } const auto precompiled_path{GetPrecompiledPath()}; const bool existed = FileUtil::Exists(precompiled_path); @@ -497,9 +599,12 @@ FileUtil::IOFile ShaderDiskCacheOpenGL::AppendPrecompiledFile() const { } if (!existed || file.GetSize() == 0) { - std::array hash{}; - std::strcpy(hash.data(), GetShaderHash().c_str()); - file.WriteArray(hash.data(), hash.size()); + const auto hash{GetShaderCacheVersionHash()}; + if (file.WriteArray(hash.data(), hash.size()) != hash.size()) { + LOG_ERROR(Render_OpenGL, "Failed to write precompiled cache version hash in path={}", + precompiled_path); + return {}; + } } return file; } diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.h b/src/video_core/renderer_opengl/gl_shader_disk_cache.h index 4bffe4307..ddcd4cf51 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.h +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.h @@ -53,15 +53,15 @@ struct BaseBindings { /// Describes a shader how it's used by the guest GPU class ShaderDiskCacheRaw { public: - explicit ShaderDiskCacheRaw(FileUtil::IOFile& file); - explicit ShaderDiskCacheRaw(u64 unique_identifier, Maxwell::ShaderProgram program_type, u32 program_code_size, u32 program_code_size_b, ProgramCode program_code, ProgramCode program_code_b); - + ShaderDiskCacheRaw(); ~ShaderDiskCacheRaw(); - void Save(FileUtil::IOFile& file) const; + bool Load(FileUtil::IOFile& file); + + bool Save(FileUtil::IOFile& file) const; u64 GetUniqueIdentifier() const { return unique_identifier; @@ -158,10 +158,10 @@ public: LoadPrecompiled(); /// Removes the transferable (and precompiled) cache file. - bool InvalidateTransferable() const; + void InvalidateTransferable() const; /// Removes the precompiled cache file. - bool InvalidatePrecompiled() const; + void InvalidatePrecompiled() const; /// Saves a raw dump to the transferable file. Checks for collisions. void SaveRaw(const ShaderDiskCacheRaw& entry); @@ -177,6 +177,22 @@ public: void SaveDump(const ShaderDiskCacheUsage& usage, GLuint program); private: + /// Loads the transferable cache. Returns empty on failure. + std::optional, + std::map>> + LoadPrecompiledFile(FileUtil::IOFile& file); + + /// Loads a decompiled cache entry from the passed file. Returns empty on failure. + std::optional LoadDecompiledEntry(FileUtil::IOFile& file); + + /// Saves a decompiled entry to the passed file. Returns true on success. + bool SaveDecompiledFile(FileUtil::IOFile& file, u64 unique_identifier, const std::string& code, + const std::vector& compressed_code, + const GLShader::ShaderEntries& entries); + + /// Returns if the cache can be used + bool IsUsable() const; + /// Opens current game's transferable file and write it's header if it doesn't exist FileUtil::IOFile AppendTransferableFile() const; @@ -203,6 +219,8 @@ private: // Stored transferable shaders std::map> transferable; + // The cache has been loaded at boot + bool tried_to_load{}; }; } // namespace OpenGL \ No newline at end of file From 7fefec585c805fa09951da11890bb90afb8a42a9 Mon Sep 17 00:00:00 2001 From: ReinUsesLisp Date: Tue, 15 Jan 2019 16:28:42 -0300 Subject: [PATCH 22/32] gl_shader_disk_cache: Pass core system as argument and guard against games without title ids --- src/core/core.cpp | 2 +- src/video_core/renderer_opengl/gl_rasterizer.cpp | 5 +++-- src/video_core/renderer_opengl/gl_rasterizer.h | 7 ++++++- src/video_core/renderer_opengl/gl_shader_cache.cpp | 3 ++- src/video_core/renderer_opengl/gl_shader_cache.h | 6 +++++- .../renderer_opengl/gl_shader_disk_cache.cpp | 14 ++++++++++---- .../renderer_opengl/gl_shader_disk_cache.h | 13 ++++++++++++- src/video_core/renderer_opengl/renderer_opengl.cpp | 6 +++--- src/video_core/renderer_opengl/renderer_opengl.h | 8 +++++++- src/video_core/video_core.cpp | 5 +++-- src/video_core/video_core.h | 7 ++++++- 11 files changed, 58 insertions(+), 18 deletions(-) diff --git a/src/core/core.cpp b/src/core/core.cpp index c8d7c442a..1d71312aa 100644 --- a/src/core/core.cpp +++ b/src/core/core.cpp @@ -123,7 +123,7 @@ struct System::Impl { Service::Init(service_manager, *virtual_filesystem); GDBStub::Init(); - renderer = VideoCore::CreateRenderer(emu_window); + renderer = VideoCore::CreateRenderer(emu_window, system); if (!renderer->Init()) { return ResultStatus::ErrorVideoCore; } diff --git a/src/video_core/renderer_opengl/gl_rasterizer.cpp b/src/video_core/renderer_opengl/gl_rasterizer.cpp index 48e003fa1..94a5058de 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer.cpp +++ b/src/video_core/renderer_opengl/gl_rasterizer.cpp @@ -100,8 +100,9 @@ struct FramebufferCacheKey { } }; -RasterizerOpenGL::RasterizerOpenGL(Core::Frontend::EmuWindow& window, ScreenInfo& info) - : res_cache{*this}, shader_cache{*this}, emu_window{window}, screen_info{info}, +RasterizerOpenGL::RasterizerOpenGL(Core::Frontend::EmuWindow& window, Core::System& system, + ScreenInfo& info) + : res_cache{*this}, shader_cache{*this, system}, emu_window{window}, screen_info{info}, buffer_cache(*this, STREAM_BUFFER_SIZE), global_cache{*this} { // Create sampler objects for (std::size_t i = 0; i < texture_samplers.size(); ++i) { diff --git a/src/video_core/renderer_opengl/gl_rasterizer.h b/src/video_core/renderer_opengl/gl_rasterizer.h index ed7091f18..ebabf80d1 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer.h +++ b/src/video_core/renderer_opengl/gl_rasterizer.h @@ -33,6 +33,10 @@ #include "video_core/renderer_opengl/gl_state.h" #include "video_core/renderer_opengl/gl_stream_buffer.h" +namespace Core { +class System; +} + namespace Core::Frontend { class EmuWindow; } @@ -45,7 +49,8 @@ struct FramebufferCacheKey; class RasterizerOpenGL : public VideoCore::RasterizerInterface { public: - explicit RasterizerOpenGL(Core::Frontend::EmuWindow& renderer, ScreenInfo& info); + explicit RasterizerOpenGL(Core::Frontend::EmuWindow& window, Core::System& system, + ScreenInfo& info); ~RasterizerOpenGL() override; void DrawArrays() override; diff --git a/src/video_core/renderer_opengl/gl_shader_cache.cpp b/src/video_core/renderer_opengl/gl_shader_cache.cpp index 6acfd1649..b2b5c2aa5 100644 --- a/src/video_core/renderer_opengl/gl_shader_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_cache.cpp @@ -342,7 +342,8 @@ ShaderDiskCacheUsage CachedShader::GetUsage(GLenum primitive_mode, return {unique_identifier, base_bindings, primitive_mode}; } -ShaderCacheOpenGL::ShaderCacheOpenGL(RasterizerOpenGL& rasterizer) : RasterizerCache{rasterizer} {} +ShaderCacheOpenGL::ShaderCacheOpenGL(RasterizerOpenGL& rasterizer, Core::System& system) + : RasterizerCache{rasterizer}, disk_cache{system} {} void ShaderCacheOpenGL::LoadDiskCache() { const auto transferable = disk_cache.LoadTransferable(); diff --git a/src/video_core/renderer_opengl/gl_shader_cache.h b/src/video_core/renderer_opengl/gl_shader_cache.h index c6a621ae3..6914127c3 100644 --- a/src/video_core/renderer_opengl/gl_shader_cache.h +++ b/src/video_core/renderer_opengl/gl_shader_cache.h @@ -20,6 +20,10 @@ #include "video_core/renderer_opengl/gl_shader_disk_cache.h" #include "video_core/renderer_opengl/gl_shader_gen.h" +namespace Core { +class System; +} // namespace Core + namespace OpenGL { class CachedShader; @@ -107,7 +111,7 @@ private: class ShaderCacheOpenGL final : public RasterizerCache { public: - explicit ShaderCacheOpenGL(RasterizerOpenGL& rasterizer); + explicit ShaderCacheOpenGL(RasterizerOpenGL& rasterizer, Core::System& system); /// Loads disk cache for the current game void LoadDiskCache(); diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp index f8bdb7779..d88fff388 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp @@ -43,9 +43,6 @@ static_assert(sizeof(BaseBindings) == 12); static_assert(sizeof(ShaderDiskCacheUsage) == 24); namespace { -std::string GetTitleID() { - return fmt::format("{:016X}", Core::CurrentProcess()->GetTitleID()); -} ShaderCacheVersionHash GetShaderCacheVersionHash() { ShaderCacheVersionHash hash{}; @@ -82,6 +79,7 @@ std::vector DecompressData(const std::vector& compressed, std::size_t un } return uncompressed; } + } // namespace ShaderDiskCacheRaw::ShaderDiskCacheRaw(u64 unique_identifier, Maxwell::ShaderProgram program_type, @@ -137,9 +135,13 @@ bool ShaderDiskCacheRaw::Save(FileUtil::IOFile& file) const { return true; } +ShaderDiskCacheOpenGL::ShaderDiskCacheOpenGL(Core::System& system) : system{system} {} + std::optional, std::vector>> ShaderDiskCacheOpenGL::LoadTransferable() { - if (!Settings::values.use_disk_shader_cache) + // Skip games without title id + const bool has_title_id = system.CurrentProcess()->GetTitleID() != 0; + if (!Settings::values.use_disk_shader_cache || !has_title_id) return {}; tried_to_load = true; @@ -643,4 +645,8 @@ std::string ShaderDiskCacheOpenGL::GetBaseDir() const { return FileUtil::GetUserPath(FileUtil::UserPath::ShaderDir) + DIR_SEP "opengl"; } +std::string ShaderDiskCacheOpenGL::GetTitleID() const { + return fmt::format("{:016X}", system.CurrentProcess()->GetTitleID()); +} + } // namespace OpenGL \ No newline at end of file diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.h b/src/video_core/renderer_opengl/gl_shader_disk_cache.h index ddcd4cf51..061c4f204 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.h +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.h @@ -18,9 +18,13 @@ #include "video_core/engines/maxwell_3d.h" #include "video_core/renderer_opengl/gl_shader_gen.h" +namespace Core { +class System; +} + namespace FileUtil { class IOFile; -} // namespace FileUtil +} namespace OpenGL { @@ -148,6 +152,8 @@ struct ShaderDiskCacheDump { class ShaderDiskCacheOpenGL { public: + explicit ShaderDiskCacheOpenGL(Core::System& system); + /// Loads transferable cache. If file has a old version or on failure, it deletes the file. std::optional, std::vector>> LoadTransferable(); @@ -217,6 +223,11 @@ private: /// Get user's shader directory path std::string GetBaseDir() const; + /// Get current game's title id + std::string GetTitleID() const; + + // Copre system + Core::System& system; // Stored transferable shaders std::map> transferable; // The cache has been loaded at boot diff --git a/src/video_core/renderer_opengl/renderer_opengl.cpp b/src/video_core/renderer_opengl/renderer_opengl.cpp index 5b09c38ea..6476a9e1a 100644 --- a/src/video_core/renderer_opengl/renderer_opengl.cpp +++ b/src/video_core/renderer_opengl/renderer_opengl.cpp @@ -98,8 +98,8 @@ static std::array MakeOrthographicMatrix(const float width, cons return matrix; } -RendererOpenGL::RendererOpenGL(Core::Frontend::EmuWindow& window) - : VideoCore::RendererBase{window} {} +RendererOpenGL::RendererOpenGL(Core::Frontend::EmuWindow& window, Core::System& system) + : VideoCore::RendererBase{window}, system{system} {} RendererOpenGL::~RendererOpenGL() = default; @@ -250,7 +250,7 @@ void RendererOpenGL::CreateRasterizer() { } // Initialize sRGB Usage OpenGLState::ClearsRGBUsed(); - rasterizer = std::make_unique(render_window, screen_info); + rasterizer = std::make_unique(render_window, system, screen_info); } void RendererOpenGL::ConfigureFramebufferTexture(TextureInfo& texture, diff --git a/src/video_core/renderer_opengl/renderer_opengl.h b/src/video_core/renderer_opengl/renderer_opengl.h index 1665018db..7e13e566b 100644 --- a/src/video_core/renderer_opengl/renderer_opengl.h +++ b/src/video_core/renderer_opengl/renderer_opengl.h @@ -12,6 +12,10 @@ #include "video_core/renderer_opengl/gl_resource_manager.h" #include "video_core/renderer_opengl/gl_state.h" +namespace Core { +class System; +} + namespace Core::Frontend { class EmuWindow; } @@ -41,7 +45,7 @@ struct ScreenInfo { class RendererOpenGL : public VideoCore::RendererBase { public: - explicit RendererOpenGL(Core::Frontend::EmuWindow& window); + explicit RendererOpenGL(Core::Frontend::EmuWindow& window, Core::System& system); ~RendererOpenGL() override; /// Swap buffers (render frame) @@ -72,6 +76,8 @@ private: void LoadColorToActiveGLTexture(u8 color_r, u8 color_g, u8 color_b, u8 color_a, const TextureInfo& texture); + Core::System& system; + OpenGLState state; // OpenGL object IDs diff --git a/src/video_core/video_core.cpp b/src/video_core/video_core.cpp index 0b8ccdd44..cb82ecf3f 100644 --- a/src/video_core/video_core.cpp +++ b/src/video_core/video_core.cpp @@ -11,8 +11,9 @@ namespace VideoCore { -std::unique_ptr CreateRenderer(Core::Frontend::EmuWindow& emu_window) { - return std::make_unique(emu_window); +std::unique_ptr CreateRenderer(Core::Frontend::EmuWindow& emu_window, + Core::System& system) { + return std::make_unique(emu_window, system); } u16 GetResolutionScaleFactor(const RendererBase& renderer) { diff --git a/src/video_core/video_core.h b/src/video_core/video_core.h index 5b373bcb1..3c583f195 100644 --- a/src/video_core/video_core.h +++ b/src/video_core/video_core.h @@ -6,6 +6,10 @@ #include +namespace Core { +class System; +} + namespace Core::Frontend { class EmuWindow; } @@ -20,7 +24,8 @@ class RendererBase; * @note The returned renderer instance is simply allocated. Its Init() * function still needs to be called to fully complete its setup. */ -std::unique_ptr CreateRenderer(Core::Frontend::EmuWindow& emu_window); +std::unique_ptr CreateRenderer(Core::Frontend::EmuWindow& emu_window, + Core::System& system); u16 GetResolutionScaleFactor(const RendererBase& renderer); From df0f31f44ea4e6945031a84f4bed7f231649fdb4 Mon Sep 17 00:00:00 2001 From: ReinUsesLisp Date: Wed, 16 Jan 2019 04:37:35 -0300 Subject: [PATCH 23/32] gl_shader_cache: Set GL_PROGRAM_SEPARABLE to dumped shaders i965 (and probably all mesa drivers) require GL_PROGRAM_SEPARABLE when using glProgramBinary. This is probably required by the standard but it's ignored by permisive proprietary drivers. --- src/video_core/renderer_opengl/gl_shader_cache.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/video_core/renderer_opengl/gl_shader_cache.cpp b/src/video_core/renderer_opengl/gl_shader_cache.cpp index b2b5c2aa5..7eeae082a 100644 --- a/src/video_core/renderer_opengl/gl_shader_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_cache.cpp @@ -405,6 +405,7 @@ CachedProgram ShaderCacheOpenGL::GeneratePrecompiledProgram( CachedProgram shader = std::make_shared(); shader->handle = glCreateProgram(); + glProgramParameteri(shader->handle, GL_PROGRAM_SEPARABLE, GL_TRUE); glProgramBinary(shader->handle, dump.binary_format, dump.binary.data(), static_cast(dump.binary.size())); From eb7324743372d3b94db9a915a5508d8293d45ecb Mon Sep 17 00:00:00 2001 From: ReinUsesLisp Date: Mon, 21 Jan 2019 16:38:23 -0300 Subject: [PATCH 24/32] gl_shader_cache: Link loading screen with disk shader cache load --- src/core/core.cpp | 2 -- src/video_core/rasterizer_interface.h | 4 ++- .../renderer_opengl/gl_rasterizer.cpp | 7 +++-- .../renderer_opengl/gl_rasterizer.h | 4 ++- .../renderer_opengl/gl_shader_cache.cpp | 29 +++++++++++++++++-- .../renderer_opengl/gl_shader_cache.h | 5 +++- src/yuzu/bootmanager.cpp | 9 ++++++ src/yuzu/bootmanager.h | 8 ++++- src/yuzu/main.cpp | 3 ++ src/yuzu_cmd/yuzu.cpp | 3 ++ 10 files changed, 62 insertions(+), 12 deletions(-) diff --git a/src/core/core.cpp b/src/core/core.cpp index 1d71312aa..1dd576c26 100644 --- a/src/core/core.cpp +++ b/src/core/core.cpp @@ -176,8 +176,6 @@ struct System::Impl { static_cast(load_result)); } - renderer->Rasterizer().LoadDiskResources(); - status = ResultStatus::Success; return status; } diff --git a/src/video_core/rasterizer_interface.h b/src/video_core/rasterizer_interface.h index bb4bc0e36..77da135a0 100644 --- a/src/video_core/rasterizer_interface.h +++ b/src/video_core/rasterizer_interface.h @@ -4,6 +4,7 @@ #pragma once +#include #include #include "common/common_types.h" #include "video_core/engines/fermi_2d.h" @@ -63,6 +64,7 @@ public: virtual void UpdatePagesCachedCount(Tegra::GPUVAddr addr, u64 size, int delta) {} /// Initialize disk cached resources for the game being emulated - virtual void LoadDiskResources() {} + virtual void LoadDiskResources(const std::atomic_bool& stop_loading = false, + const DiskResourceLoadCallback& callback = {}) {} }; } // namespace VideoCore diff --git a/src/video_core/renderer_opengl/gl_rasterizer.cpp b/src/video_core/renderer_opengl/gl_rasterizer.cpp index 94a5058de..974ca6a20 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer.cpp +++ b/src/video_core/renderer_opengl/gl_rasterizer.cpp @@ -449,7 +449,7 @@ static constexpr auto RangeFromInterval(Map& map, const Interval& interval) { return boost::make_iterator_range(map.equal_range(interval)); } -void RasterizerOpenGL::UpdatePagesCachedCount(VAddr addr, u64 size, int delta) { +void RasterizerOpenGL::UpdatePagesCachedCount(Tegra::GPUVAddr addr, u64 size, int delta) { const u64 page_start{addr >> Memory::PAGE_BITS}; const u64 page_end{(addr + size + Memory::PAGE_SIZE - 1) >> Memory::PAGE_BITS}; @@ -479,8 +479,9 @@ void RasterizerOpenGL::UpdatePagesCachedCount(VAddr addr, u64 size, int delta) { cached_pages.add({pages_interval, delta}); } -void RasterizerOpenGL::LoadDiskResources() { - shader_cache.LoadDiskCache(); +void RasterizerOpenGL::LoadDiskResources(const std::atomic_bool& stop_loading, + const VideoCore::DiskResourceLoadCallback& callback) { + shader_cache.LoadDiskCache(stop_loading, callback); } std::pair RasterizerOpenGL::ConfigureFramebuffers( diff --git a/src/video_core/renderer_opengl/gl_rasterizer.h b/src/video_core/renderer_opengl/gl_rasterizer.h index ebabf80d1..f3b607f4d 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer.h +++ b/src/video_core/renderer_opengl/gl_rasterizer.h @@ -5,6 +5,7 @@ #pragma once #include +#include #include #include #include @@ -65,7 +66,8 @@ public: u32 pixel_stride) override; bool AccelerateDrawBatch(bool is_indexed) override; void UpdatePagesCachedCount(Tegra::GPUVAddr addr, u64 size, int delta) override; - void LoadDiskResources() override; + void LoadDiskResources(const std::atomic_bool& stop_loading, + const VideoCore::DiskResourceLoadCallback& callback) override; /// Maximum supported size that a constbuffer can have in bytes. static constexpr std::size_t MaxConstbufferSize = 0x10000; diff --git a/src/video_core/renderer_opengl/gl_shader_cache.cpp b/src/video_core/renderer_opengl/gl_shader_cache.cpp index 7eeae082a..e0e624e6f 100644 --- a/src/video_core/renderer_opengl/gl_shader_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_cache.cpp @@ -345,7 +345,8 @@ ShaderDiskCacheUsage CachedShader::GetUsage(GLenum primitive_mode, ShaderCacheOpenGL::ShaderCacheOpenGL(RasterizerOpenGL& rasterizer, Core::System& system) : RasterizerCache{rasterizer}, disk_cache{system} {} -void ShaderCacheOpenGL::LoadDiskCache() { +void ShaderCacheOpenGL::LoadDiskCache(const std::atomic_bool& stop_loading, + const VideoCore::DiskResourceLoadCallback& callback) { const auto transferable = disk_cache.LoadTransferable(); if (!transferable) { return; @@ -355,10 +356,18 @@ void ShaderCacheOpenGL::LoadDiskCache() { auto [decompiled, dumps] = disk_cache.LoadPrecompiled(); const auto supported_formats{GetSupportedFormats()}; - const auto unspecialized{GenerateUnspecializedShaders(raws, decompiled)}; + const auto unspecialized{ + GenerateUnspecializedShaders(stop_loading, callback, raws, decompiled)}; + if (stop_loading) + return; // Build shaders + if (callback) + callback(VideoCore::LoadCallbackStage::Build, 0, usages.size()); for (std::size_t i = 0; i < usages.size(); ++i) { + if (stop_loading) + return; + const auto& usage{usages[i]}; LOG_INFO(Render_OpenGL, "Building shader {:016x} ({} of {})", usage.unique_identifier, i + 1, usages.size()); @@ -381,6 +390,9 @@ void ShaderCacheOpenGL::LoadDiskCache() { usage.bindings, usage.primitive, true); } precompiled_programs.insert({usage, std::move(shader)}); + + if (callback) + callback(VideoCore::LoadCallbackStage::Build, i + 1, usages.size()); } // TODO(Rodrigo): Do state tracking for transferable shaders and do a dummy draw before @@ -420,11 +432,19 @@ CachedProgram ShaderCacheOpenGL::GeneratePrecompiledProgram( } std::map ShaderCacheOpenGL::GenerateUnspecializedShaders( + const std::atomic_bool& stop_loading, const VideoCore::DiskResourceLoadCallback& callback, const std::vector& raws, const std::map& decompiled) { std::map unspecialized; - for (const auto& raw : raws) { + if (callback) + callback(VideoCore::LoadCallbackStage::Decompile, 0, raws.size()); + + for (std::size_t i = 0; i < raws.size(); ++i) { + if (stop_loading) + return {}; + + const auto& raw{raws[i]}; const u64 unique_identifier = raw.GetUniqueIdentifier(); const u64 calculated_hash = GetUniqueIdentifier(raw.GetProgramType(), raw.GetProgramCode(), raw.GetProgramCodeB()); @@ -454,6 +474,9 @@ std::map ShaderCacheOpenGL::GenerateUnspecializedShade unspecialized.insert( {raw.GetUniqueIdentifier(), {std::move(result.first), std::move(result.second), raw.GetProgramType()}}); + + if (callback) + callback(VideoCore::LoadCallbackStage::Decompile, i, raws.size()); } return unspecialized; } diff --git a/src/video_core/renderer_opengl/gl_shader_cache.h b/src/video_core/renderer_opengl/gl_shader_cache.h index 6914127c3..9c6b19fc3 100644 --- a/src/video_core/renderer_opengl/gl_shader_cache.h +++ b/src/video_core/renderer_opengl/gl_shader_cache.h @@ -15,6 +15,7 @@ #include "common/assert.h" #include "common/common_types.h" #include "video_core/rasterizer_cache.h" +#include "video_core/renderer_base.h" #include "video_core/renderer_opengl/gl_resource_manager.h" #include "video_core/renderer_opengl/gl_shader_decompiler.h" #include "video_core/renderer_opengl/gl_shader_disk_cache.h" @@ -114,13 +115,15 @@ public: explicit ShaderCacheOpenGL(RasterizerOpenGL& rasterizer, Core::System& system); /// Loads disk cache for the current game - void LoadDiskCache(); + void LoadDiskCache(const std::atomic_bool& stop_loading, + const VideoCore::DiskResourceLoadCallback& callback); /// Gets the current specified shader stage program Shader GetStageProgram(Maxwell::ShaderProgram program); private: std::map GenerateUnspecializedShaders( + const std::atomic_bool& stop_loading, const VideoCore::DiskResourceLoadCallback& callback, const std::vector& raws, const std::map& decompiled); diff --git a/src/yuzu/bootmanager.cpp b/src/yuzu/bootmanager.cpp index f74cb693a..73b04b749 100644 --- a/src/yuzu/bootmanager.cpp +++ b/src/yuzu/bootmanager.cpp @@ -29,6 +29,15 @@ void EmuThread::run() { stop_run = false; + emit LoadProgress(VideoCore::LoadCallbackStage::Prepare, 0, 0); + + Core::System::GetInstance().Renderer().Rasterizer().LoadDiskResources( + stop_run, [this](VideoCore::LoadCallbackStage stage, std::size_t value, std::size_t total) { + emit LoadProgress(stage, value, total); + }); + + emit LoadProgress(VideoCore::LoadCallbackStage::Complete, 0, 0); + // holds whether the cpu was running during the last iteration, // so that the DebugModeLeft signal can be emitted before the // next execution step diff --git a/src/yuzu/bootmanager.h b/src/yuzu/bootmanager.h index d1f37e503..7226e690e 100644 --- a/src/yuzu/bootmanager.h +++ b/src/yuzu/bootmanager.h @@ -22,6 +22,10 @@ class GGLWidgetInternal; class GMainWindow; class GRenderWindow; +namespace VideoCore { +enum class LoadCallbackStage; +} + class EmuThread : public QThread { Q_OBJECT @@ -75,7 +79,7 @@ public: private: bool exec_step = false; bool running = false; - std::atomic stop_run{false}; + std::atomic_bool stop_run{false}; std::mutex running_mutex; std::condition_variable running_cv; @@ -101,6 +105,8 @@ signals: void DebugModeLeft(); void ErrorThrown(Core::System::ResultStatus, std::string); + + void LoadProgress(VideoCore::LoadCallbackStage stage, std::size_t value, std::size_t total); }; class GRenderWindow : public QWidget, public Core::Frontend::EmuWindow { diff --git a/src/yuzu/main.cpp b/src/yuzu/main.cpp index 485e29de2..1d460c189 100644 --- a/src/yuzu/main.cpp +++ b/src/yuzu/main.cpp @@ -887,6 +887,9 @@ void GMainWindow::BootGame(const QString& filename) { connect(emu_thread.get(), &EmuThread::DebugModeLeft, waitTreeWidget, &WaitTreeWidget::OnDebugModeLeft, Qt::BlockingQueuedConnection); + connect(emu_thread.get(), &EmuThread::LoadProgress, loading_screen, + &LoadingScreen::OnLoadProgress, Qt::QueuedConnection); + // Update the GUI if (ui.action_Single_Window_Mode->isChecked()) { game_list->hide(); diff --git a/src/yuzu_cmd/yuzu.cpp b/src/yuzu_cmd/yuzu.cpp index 806127b12..c34b5467f 100644 --- a/src/yuzu_cmd/yuzu.cpp +++ b/src/yuzu_cmd/yuzu.cpp @@ -28,6 +28,7 @@ #include "core/loader/loader.h" #include "core/settings.h" #include "core/telemetry_session.h" +#include "video_core/renderer_base.h" #include "yuzu_cmd/config.h" #include "yuzu_cmd/emu_window/emu_window_sdl2.h" @@ -217,6 +218,8 @@ int main(int argc, char** argv) { Core::Telemetry().AddField(Telemetry::FieldType::App, "Frontend", "SDL"); + system.Renderer().Rasterizer().LoadDiskResources(); + while (emu_window->IsOpen()) { system.RunLoop(); } From bd928e70edb5ae96ecdfd04edf217e7f504e6f7d Mon Sep 17 00:00:00 2001 From: ReinUsesLisp Date: Mon, 21 Jan 2019 16:38:37 -0300 Subject: [PATCH 25/32] loading_screen: Unchunk progress bar --- src/yuzu/loading_screen.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/yuzu/loading_screen.cpp b/src/yuzu/loading_screen.cpp index 907aac4f1..86f6d0165 100644 --- a/src/yuzu/loading_screen.cpp +++ b/src/yuzu/loading_screen.cpp @@ -43,6 +43,7 @@ QProgressBar { } QProgressBar::chunk { background-color: #0ab9e6; + width: 1px; })"; constexpr const char PROGRESSBAR_STYLE_BUILD[] = R"( @@ -53,7 +54,8 @@ QProgressBar { padding: 2px; } QProgressBar::chunk { - background-color: #ff3c28; + background-color: #ff3c28; + width: 1px; })"; constexpr const char PROGRESSBAR_STYLE_COMPLETE[] = R"( From 4ffb4872517e07ff712028343d6be636e6eaa3fc Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 21 Jan 2019 21:10:37 -0300 Subject: [PATCH 26/32] cmake: Fixup application string Co-Authored-By: ReinUsesLisp --- CMakeModules/GenerateSCMRev.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeModules/GenerateSCMRev.cmake b/CMakeModules/GenerateSCMRev.cmake index e82ad204d..743b14407 100644 --- a/CMakeModules/GenerateSCMRev.cmake +++ b/CMakeModules/GenerateSCMRev.cmake @@ -25,7 +25,7 @@ if ($ENV{CI}) set(BUILD_TAG $ENV{APPVEYOR_REPO_TAG_NAME}) endif() # regex capture the string nightly or canary into CMAKE_MATCH_1 - string(REGEX MATCH "citra-emu/citra-?(.*)" OUTVAR ${BUILD_REPOSITORY}) + string(REGEX MATCH "yuzu-emu/yuzu-?(.*)" OUTVAR ${BUILD_REPOSITORY}) if (${CMAKE_MATCH_COUNT} GREATER 0) # capitalize the first letter of each word in the repo name. string(REPLACE "-" ";" REPO_NAME_LIST ${CMAKE_MATCH_1}) @@ -98,4 +98,4 @@ foreach (F IN LISTS HASH_FILES) set(COMBINED "${COMBINED}${TMP}") endforeach() string(MD5 SHADER_CACHE_VERSION "${COMBINED}") -configure_file("${SRC_DIR}/src/common/scm_rev.cpp.in" "scm_rev.cpp" @ONLY) \ No newline at end of file +configure_file("${SRC_DIR}/src/common/scm_rev.cpp.in" "scm_rev.cpp" @ONLY) From e147ed4fc039d4e1367567d2992300d5f7b61c55 Mon Sep 17 00:00:00 2001 From: ReinUsesLisp Date: Thu, 31 Jan 2019 16:44:11 -0300 Subject: [PATCH 27/32] gl_shader_cache: Fixup GLSL unique identifiers --- src/video_core/renderer_opengl/gl_shader_cache.cpp | 4 ++-- src/video_core/renderer_opengl/gl_shader_gen.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/video_core/renderer_opengl/gl_shader_cache.cpp b/src/video_core/renderer_opengl/gl_shader_cache.cpp index e0e624e6f..bbcd15867 100644 --- a/src/video_core/renderer_opengl/gl_shader_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_cache.cpp @@ -136,12 +136,12 @@ u64 GetUniqueIdentifier(Maxwell::ShaderProgram program_type, const ProgramCode& /// Creates an unspecialized program from code streams GLShader::ProgramResult CreateProgram(Maxwell::ShaderProgram program_type, ProgramCode program_code, ProgramCode program_code_b) { - GLShader::ShaderSetup setup(std::move(program_code)); + GLShader::ShaderSetup setup(program_code); if (program_type == Maxwell::ShaderProgram::VertexA) { // VertexB is always enabled, so when VertexA is enabled, we have two vertex shaders. // Conventional HW does not support this, so we combine VertexA and VertexB into one // stage here. - setup.SetProgramB(std::move(program_code_b)); + setup.SetProgramB(program_code_b); } setup.program.unique_identifier = GetUniqueIdentifier(program_type, program_code, program_code_b); diff --git a/src/video_core/renderer_opengl/gl_shader_gen.h b/src/video_core/renderer_opengl/gl_shader_gen.h index fd3105de3..fba8e681b 100644 --- a/src/video_core/renderer_opengl/gl_shader_gen.h +++ b/src/video_core/renderer_opengl/gl_shader_gen.h @@ -29,7 +29,7 @@ struct ShaderSetup { } program; /// Used in scenarios where we have a dual vertex shaders - void SetProgramB(ProgramCode&& program_b) { + void SetProgramB(ProgramCode program_b) { program.code_b = std::move(program_b); has_program_b = true; } From e6a224530454987aa7e4dfabe2f4d9b768b99a44 Mon Sep 17 00:00:00 2001 From: ReinUsesLisp Date: Sat, 2 Feb 2019 21:14:36 -0300 Subject: [PATCH 28/32] gl_shader_disk_cache: Use unordered containers --- .../renderer_opengl/gl_shader_cache.cpp | 6 +- .../renderer_opengl/gl_shader_cache.h | 20 ++--- .../renderer_opengl/gl_shader_disk_cache.cpp | 12 +-- .../renderer_opengl/gl_shader_disk_cache.h | 82 ++++++++++--------- 4 files changed, 64 insertions(+), 56 deletions(-) diff --git a/src/video_core/renderer_opengl/gl_shader_cache.cpp b/src/video_core/renderer_opengl/gl_shader_cache.cpp index bbcd15867..4883e4f62 100644 --- a/src/video_core/renderer_opengl/gl_shader_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_cache.cpp @@ -431,11 +431,11 @@ CachedProgram ShaderCacheOpenGL::GeneratePrecompiledProgram( return shader; } -std::map ShaderCacheOpenGL::GenerateUnspecializedShaders( +std::unordered_map ShaderCacheOpenGL::GenerateUnspecializedShaders( const std::atomic_bool& stop_loading, const VideoCore::DiskResourceLoadCallback& callback, const std::vector& raws, - const std::map& decompiled) { - std::map unspecialized; + const std::unordered_map& decompiled) { + std::unordered_map unspecialized; if (callback) callback(VideoCore::LoadCallbackStage::Decompile, 0, raws.size()); diff --git a/src/video_core/renderer_opengl/gl_shader_cache.h b/src/video_core/renderer_opengl/gl_shader_cache.h index 9c6b19fc3..97eed192f 100644 --- a/src/video_core/renderer_opengl/gl_shader_cache.h +++ b/src/video_core/renderer_opengl/gl_shader_cache.h @@ -5,10 +5,10 @@ #pragma once #include -#include #include #include #include +#include #include @@ -34,8 +34,8 @@ struct UnspecializedShader; using Shader = std::shared_ptr; using CachedProgram = std::shared_ptr; using Maxwell = Tegra::Engines::Maxwell3D::Regs; -using PrecompiledPrograms = std::map; -using PrecompiledShaders = std::map; +using PrecompiledPrograms = std::unordered_map; +using PrecompiledShaders = std::unordered_map; class CachedShader final : public RasterizerCacheObject { public: @@ -102,12 +102,12 @@ private: std::string code; - std::map programs; - std::map geometry_programs; + std::unordered_map programs; + std::unordered_map geometry_programs; - std::map cbuf_resource_cache; - std::map gmem_resource_cache; - std::map uniform_cache; + std::unordered_map cbuf_resource_cache; + std::unordered_map gmem_resource_cache; + std::unordered_map uniform_cache; }; class ShaderCacheOpenGL final : public RasterizerCache { @@ -122,10 +122,10 @@ public: Shader GetStageProgram(Maxwell::ShaderProgram program); private: - std::map GenerateUnspecializedShaders( + std::unordered_map GenerateUnspecializedShaders( const std::atomic_bool& stop_loading, const VideoCore::DiskResourceLoadCallback& callback, const std::vector& raws, - const std::map& decompiled); + const std::unordered_map& decompiled); CachedProgram GeneratePrecompiledProgram(const ShaderDiskCacheDump& dump, const std::set& supported_formats); diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp index d88fff388..554d0dfb9 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp @@ -211,8 +211,8 @@ ShaderDiskCacheOpenGL::LoadTransferable() { return {{raws, usages}}; } -std::pair, - std::map> +std::pair, + std::unordered_map> ShaderDiskCacheOpenGL::LoadPrecompiled() { if (!IsUsable()) return {}; @@ -236,8 +236,8 @@ ShaderDiskCacheOpenGL::LoadPrecompiled() { return *result; } -std::optional, - std::map>> +std::optional, + std::unordered_map>> ShaderDiskCacheOpenGL::LoadPrecompiledFile(FileUtil::IOFile& file) { ShaderCacheVersionHash file_hash{}; if (file.ReadArray(file_hash.data(), file_hash.size()) != file_hash.size()) { @@ -248,8 +248,8 @@ ShaderDiskCacheOpenGL::LoadPrecompiledFile(FileUtil::IOFile& file) { return {}; } - std::map decompiled; - std::map dumps; + std::unordered_map decompiled; + std::unordered_map dumps; while (file.Tell() < file.GetSize()) { PrecompiledEntryKind kind{}; if (file.ReadBytes(&kind, sizeof(u32)) != sizeof(u32)) { diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.h b/src/video_core/renderer_opengl/gl_shader_disk_cache.h index 061c4f204..6be0c0547 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.h +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.h @@ -5,9 +5,10 @@ #pragma once #include -#include #include #include +#include +#include #include #include @@ -37,23 +38,54 @@ struct BaseBindings { u32 gmem{}; u32 sampler{}; - bool operator<(const BaseBindings& rhs) const { - return Tie() < rhs.Tie(); - } - bool operator==(const BaseBindings& rhs) const { - return Tie() == rhs.Tie(); + return std::tie(cbuf, gmem, sampler) == std::tie(rhs.cbuf, rhs.gmem, rhs.sampler); } bool operator!=(const BaseBindings& rhs) const { return !operator==(rhs); } +}; - std::tuple Tie() const { - return std::tie(cbuf, gmem, sampler); +/// Describes how a shader is used +struct ShaderDiskCacheUsage { + u64 unique_identifier{}; + BaseBindings bindings; + GLenum primitive{}; + + bool operator==(const ShaderDiskCacheUsage& rhs) const { + return std::tie(unique_identifier, bindings, primitive) == + std::tie(rhs.unique_identifier, rhs.bindings, rhs.primitive); + } + + bool operator!=(const ShaderDiskCacheUsage& rhs) const { + return !operator==(rhs); } }; +} // namespace OpenGL + +namespace std { + +template <> +struct hash { + std::size_t operator()(const OpenGL::BaseBindings& bindings) const { + return bindings.cbuf | bindings.gmem << 8 | bindings.sampler << 16; + } +}; + +template <> +struct hash { + std::size_t operator()(const OpenGL::ShaderDiskCacheUsage& usage) const { + return static_cast(usage.unique_identifier) ^ + std::hash()(usage.bindings) ^ usage.primitive << 16; + } +}; + +} // namespace std + +namespace OpenGL { + /// Describes a shader how it's used by the guest GPU class ShaderDiskCacheRaw { public: @@ -114,30 +146,6 @@ private: ProgramCode program_code_b; }; -/// Describes how a shader is used -struct ShaderDiskCacheUsage { - bool operator<(const ShaderDiskCacheUsage& rhs) const { - return Tie() < rhs.Tie(); - } - - bool operator==(const ShaderDiskCacheUsage& rhs) const { - return Tie() == rhs.Tie(); - } - - bool operator!=(const ShaderDiskCacheUsage& rhs) const { - return !operator==(rhs); - } - - u64 unique_identifier{}; - BaseBindings bindings; - GLenum primitive{}; - -private: - std::tuple Tie() const { - return std::tie(unique_identifier, bindings, primitive); - } -}; - /// Contains decompiled data from a shader struct ShaderDiskCacheDecompiled { std::string code; @@ -159,8 +167,8 @@ public: LoadTransferable(); /// Loads current game's precompiled cache. Invalidates on failure. - std::pair, - std::map> + std::pair, + std::unordered_map> LoadPrecompiled(); /// Removes the transferable (and precompiled) cache file. @@ -184,8 +192,8 @@ public: private: /// Loads the transferable cache. Returns empty on failure. - std::optional, - std::map>> + std::optional, + std::unordered_map>> LoadPrecompiledFile(FileUtil::IOFile& file); /// Loads a decompiled cache entry from the passed file. Returns empty on failure. @@ -229,7 +237,7 @@ private: // Copre system Core::System& system; // Stored transferable shaders - std::map> transferable; + std::map> transferable; // The cache has been loaded at boot bool tried_to_load{}; }; From 8ff2ce520751b2eb833ded21f9f66f89059ff829 Mon Sep 17 00:00:00 2001 From: Frederic L Date: Sun, 3 Feb 2019 01:15:46 -0300 Subject: [PATCH 29/32] cmake: Use CMAKE_COMMAND instead of "cmake" Co-Authored-By: ReinUsesLisp --- src/common/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index f38c0fee9..4b6ddf894 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -4,7 +4,7 @@ # like this allows for much better caching since it doesn't force the user to recompile binary shaders every update set(VIDEO_CORE "${CMAKE_SOURCE_DIR}/src/video_core") add_custom_command(OUTPUT scm_rev.cpp - COMMAND cmake -DSRC_DIR="${CMAKE_SOURCE_DIR}" -P "${CMAKE_SOURCE_DIR}/CMakeModules/GenerateSCMRev.cmake" + COMMAND ${CMAKE_COMMAND} -DSRC_DIR="${CMAKE_SOURCE_DIR}" -P "${CMAKE_SOURCE_DIR}/CMakeModules/GenerateSCMRev.cmake" DEPENDS # WARNING! It was too much work to try and make a common location for this list, # so if you need to change it, please update CMakeModules/GenerateSCMRev.cmake as well From 9f0b247cf6776311d0826b0983369e9c248a6c55 Mon Sep 17 00:00:00 2001 From: Frederic L Date: Sun, 3 Feb 2019 01:21:18 -0300 Subject: [PATCH 30/32] gl_shader_disk_cache: Consider compressed size zero as an error Co-Authored-By: ReinUsesLisp --- src/video_core/renderer_opengl/gl_shader_disk_cache.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp index 554d0dfb9..67fcf3061 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp @@ -59,7 +59,7 @@ std::vector CompressData(const T* source, std::size_t source_size) { const int compressed_size = LZ4_compress_default(reinterpret_cast(source), reinterpret_cast(compressed.data()), source_size_int, max_compressed_size); - if (compressed_size < 0) { + if (compressed_size <= 0) { // Compression failed return {}; } @@ -649,4 +649,4 @@ std::string ShaderDiskCacheOpenGL::GetTitleID() const { return fmt::format("{:016X}", system.CurrentProcess()->GetTitleID()); } -} // namespace OpenGL \ No newline at end of file +} // namespace OpenGL From d0ac62440395566f8a68a7b6d8cb2038094aba7d Mon Sep 17 00:00:00 2001 From: Frederic L Date: Sun, 3 Feb 2019 01:22:01 -0300 Subject: [PATCH 31/32] gl_shader_disk_cache: Check LZ4 size limit Co-Authored-By: ReinUsesLisp --- src/video_core/renderer_opengl/gl_shader_disk_cache.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp index 67fcf3061..81882822b 100644 --- a/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_disk_cache.cpp @@ -53,6 +53,10 @@ ShaderCacheVersionHash GetShaderCacheVersionHash() { template std::vector CompressData(const T* source, std::size_t source_size) { + if (source_size > LZ4_MAX_INPUT_SIZE) { + // Source size exceeds LZ4 maximum input size + return {}; + } const auto source_size_int = static_cast(source_size); const int max_compressed_size = LZ4_compressBound(source_size_int); std::vector compressed(max_compressed_size); From dfd14618f7cde84caf15f3b56ee14800f2509ebc Mon Sep 17 00:00:00 2001 From: ReinUsesLisp Date: Wed, 6 Feb 2019 21:55:14 -0300 Subject: [PATCH 32/32] cmake: Fix title bar issue --- CMakeModules/GenerateSCMRev.cmake | 9 +-------- src/common/CMakeLists.txt | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/CMakeModules/GenerateSCMRev.cmake b/CMakeModules/GenerateSCMRev.cmake index 743b14407..78728e08b 100644 --- a/CMakeModules/GenerateSCMRev.cmake +++ b/CMakeModules/GenerateSCMRev.cmake @@ -16,14 +16,7 @@ get_timestamp(BUILD_DATE) # Also if this is a CI build, add the build name (ie: Nightly, Canary) to the scm_rev file as well set(REPO_NAME "") set(BUILD_VERSION "0") -if ($ENV{CI}) - if ($ENV{TRAVIS}) - set(BUILD_REPOSITORY $ENV{TRAVIS_REPO_SLUG}) - set(BUILD_TAG $ENV{TRAVIS_TAG}) - elseif($ENV{APPVEYOR}) - set(BUILD_REPOSITORY $ENV{APPVEYOR_REPO_NAME}) - set(BUILD_TAG $ENV{APPVEYOR_REPO_TAG_NAME}) - endif() +if (BUILD_REPOSITORY) # regex capture the string nightly or canary into CMAKE_MATCH_1 string(REGEX MATCH "yuzu-emu/yuzu-?(.*)" OUTVAR ${BUILD_REPOSITORY}) if (${CMAKE_MATCH_COUNT} GREATER 0) diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 4b6ddf894..bdd885273 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -3,8 +3,21 @@ # could affect the result, but much more unlikely than the following files. Keeping a list of files # like this allows for much better caching since it doesn't force the user to recompile binary shaders every update set(VIDEO_CORE "${CMAKE_SOURCE_DIR}/src/video_core") +if (DEFINED ENV{CI}) + if (DEFINED ENV{TRAVIS}) + set(BUILD_REPOSITORY $ENV{TRAVIS_REPO_SLUG}) + set(BUILD_TAG $ENV{TRAVIS_TAG}) + elseif(DEFINED ENV{APPVEYOR}) + set(BUILD_REPOSITORY $ENV{APPVEYOR_REPO_NAME}) + set(BUILD_TAG $ENV{APPVEYOR_REPO_TAG_NAME}) + endif() +endif() add_custom_command(OUTPUT scm_rev.cpp - COMMAND ${CMAKE_COMMAND} -DSRC_DIR="${CMAKE_SOURCE_DIR}" -P "${CMAKE_SOURCE_DIR}/CMakeModules/GenerateSCMRev.cmake" + COMMAND ${CMAKE_COMMAND} + -DSRC_DIR="${CMAKE_SOURCE_DIR}" + -DBUILD_REPOSITORY="${BUILD_REPOSITORY}" + -DBUILD_TAG="${BUILD_TAG}" + -P "${CMAKE_SOURCE_DIR}/CMakeModules/GenerateSCMRev.cmake" DEPENDS # WARNING! It was too much work to try and make a common location for this list, # so if you need to change it, please update CMakeModules/GenerateSCMRev.cmake as well