From 10aac376d1d0fccb5b993bd3f95db42de0ceba12 Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Thu, 9 Aug 2018 20:43:45 -0400 Subject: [PATCH 01/28] common: Move hex string processing to separate file --- src/common/CMakeLists.txt | 2 ++ src/common/hex_util.cpp | 27 +++++++++++++++++++++++++++ src/common/hex_util.h | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+) create mode 100644 src/common/hex_util.cpp create mode 100644 src/common/hex_util.h diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index d5d4f6f82..2ad456864 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -40,6 +40,8 @@ add_library(common STATIC file_util.cpp file_util.h hash.h + hex_util.cpp + hex_util.h logging/backend.cpp logging/backend.h logging/filter.cpp diff --git a/src/common/hex_util.cpp b/src/common/hex_util.cpp new file mode 100644 index 000000000..ae17c89d4 --- /dev/null +++ b/src/common/hex_util.cpp @@ -0,0 +1,27 @@ +// Copyright 2013 Dolphin Emulator Project / 2014 Citra Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#include "common/hex_util.h" + +u8 ToHexNibble(char c1) { + if (c1 >= 65 && c1 <= 70) + return c1 - 55; + if (c1 >= 97 && c1 <= 102) + return c1 - 87; + if (c1 >= 48 && c1 <= 57) + return c1 - 48; + throw std::logic_error("Invalid hex digit"); +} + +std::array operator""_array16(const char* str, size_t len) { + if (len != 32) + throw std::logic_error("Not of correct size."); + return HexStringToArray<16>(str); +} + +std::array operator""_array32(const char* str, size_t len) { + if (len != 64) + throw std::logic_error("Not of correct size."); + return HexStringToArray<32>(str); +} diff --git a/src/common/hex_util.h b/src/common/hex_util.h new file mode 100644 index 000000000..f16c58aab --- /dev/null +++ b/src/common/hex_util.h @@ -0,0 +1,35 @@ +// Copyright 2013 Dolphin Emulator Project / 2014 Citra Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#pragma once + +#include +#include +#include "common/common_types.h" + +u8 ToHexNibble(char c1); + +template +std::array HexStringToArray(std::string_view str) { + std::array out{}; + if constexpr (le) { + for (size_t i = 2 * Size - 2; i <= 2 * Size; i -= 2) + out[i / 2] = (ToHexNibble(str[i]) << 4) | ToHexNibble(str[i + 1]); + } else { + for (size_t i = 0; i < 2 * Size; i += 2) + out[i / 2] = (ToHexNibble(str[i]) << 4) | ToHexNibble(str[i + 1]); + } + return out; +} + +template +std::string HexArrayToString(std::array array, bool upper = true) { + std::string out; + for (u8 c : array) + out += fmt::format(upper ? "{:02X}" : "{:02x}", c); + return out; +} + +std::array operator"" _array16(const char* str, size_t len); +std::array operator"" _array32(const char* str, size_t len); From b70a831608caf78831e23575aa2fca19bf0d3491 Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Thu, 9 Aug 2018 20:44:21 -0400 Subject: [PATCH 02/28] file_util: Add getter for NAND registration directory --- src/common/file_util.cpp | 6 ++++++ src/common/file_util.h | 2 ++ 2 files changed, 8 insertions(+) diff --git a/src/common/file_util.cpp b/src/common/file_util.cpp index 3ce590062..b30a67ff9 100644 --- a/src/common/file_util.cpp +++ b/src/common/file_util.cpp @@ -750,6 +750,12 @@ std::string GetHactoolConfigurationPath() { #endif } +std::string GetNANDRegistrationDir(bool system) { + if (system) + return GetUserPath(UserPath::NANDDir) + "system/Contents/registered/"; + return GetUserPath(UserPath::NANDDir) + "user/Contents/registered/"; +} + size_t WriteStringToFile(bool text_file, const std::string& str, const char* filename) { return FileUtil::IOFile(filename, text_file ? "w" : "wb").WriteBytes(str.data(), str.size()); } diff --git a/src/common/file_util.h b/src/common/file_util.h index 2711872ae..2f13d0b6b 100644 --- a/src/common/file_util.h +++ b/src/common/file_util.h @@ -129,6 +129,8 @@ const std::string& GetUserPath(UserPath path, const std::string& new_path = ""); std::string GetHactoolConfigurationPath(); +std::string GetNANDRegistrationDir(bool system = false); + // Returns the path to where the sys file are std::string GetSysDirectory(); From a27ec24c0f83d346218e0301f80398209d30ffcb Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Thu, 9 Aug 2018 20:45:04 -0400 Subject: [PATCH 03/28] crypto: Remove hex utilities from key_manager Move to hex_util.h in common --- src/core/crypto/key_manager.cpp | 35 ++------------------------------- src/core/crypto/key_manager.h | 3 --- 2 files changed, 2 insertions(+), 36 deletions(-) diff --git a/src/core/crypto/key_manager.cpp b/src/core/crypto/key_manager.cpp index fc45e7ab5..94d92579f 100644 --- a/src/core/crypto/key_manager.cpp +++ b/src/core/crypto/key_manager.cpp @@ -10,44 +10,13 @@ #include #include "common/common_paths.h" #include "common/file_util.h" +#include "common/hex_util.h" +#include "common/logging/log.h" #include "core/crypto/key_manager.h" #include "core/settings.h" namespace Core::Crypto { -static u8 ToHexNibble(char c1) { - if (c1 >= 65 && c1 <= 70) - return c1 - 55; - if (c1 >= 97 && c1 <= 102) - return c1 - 87; - if (c1 >= 48 && c1 <= 57) - return c1 - 48; - throw std::logic_error("Invalid hex digit"); -} - -template -static std::array HexStringToArray(std::string_view str) { - std::array out{}; - for (size_t i = 0; i < 2 * Size; i += 2) { - auto d1 = str[i]; - auto d2 = str[i + 1]; - out[i / 2] = (ToHexNibble(d1) << 4) | ToHexNibble(d2); - } - return out; -} - -std::array operator""_array16(const char* str, size_t len) { - if (len != 32) - throw std::logic_error("Not of correct size."); - return HexStringToArray<16>(str); -} - -std::array operator""_array32(const char* str, size_t len) { - if (len != 64) - throw std::logic_error("Not of correct size."); - return HexStringToArray<32>(str); -} - KeyManager::KeyManager() { // Initialize keys const std::string hactool_keys_dir = FileUtil::GetHactoolConfigurationPath(); diff --git a/src/core/crypto/key_manager.h b/src/core/crypto/key_manager.h index c4c53cefc..0c62d4421 100644 --- a/src/core/crypto/key_manager.h +++ b/src/core/crypto/key_manager.h @@ -87,9 +87,6 @@ struct hash> { namespace Core::Crypto { -std::array operator"" _array16(const char* str, size_t len); -std::array operator"" _array32(const char* str, size_t len); - class KeyManager { public: KeyManager(); From 42114e1df49ff65f09865b53395e01858ca4929e Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Thu, 9 Aug 2018 20:46:41 -0400 Subject: [PATCH 04/28] vfs: Add ConcatenatedVfsFile --- src/core/file_sys/vfs_concat.cpp | 93 ++++++++++++++++++++++++++++++++ src/core/file_sys/vfs_concat.h | 41 ++++++++++++++ 2 files changed, 134 insertions(+) create mode 100644 src/core/file_sys/vfs_concat.cpp create mode 100644 src/core/file_sys/vfs_concat.h diff --git a/src/core/file_sys/vfs_concat.cpp b/src/core/file_sys/vfs_concat.cpp new file mode 100644 index 000000000..1d439e0a4 --- /dev/null +++ b/src/core/file_sys/vfs_concat.cpp @@ -0,0 +1,93 @@ +// Copyright 2018 yuzu emulator team +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#include +#include + +#include "core/file_sys/vfs_concat.h" + +namespace FileSys { + +VirtualFile ConcatenateFiles(std::vector files, std::string_view name) { + if (files.empty()) + return nullptr; + if (files.size() == 1) + return files[0]; + + return std::shared_ptr(new ConcatenatedVfsFile(std::move(files), name)); +} + +ConcatenatedVfsFile::ConcatenatedVfsFile(std::vector files_, std::string_view name) + : name(name) { + size_t next_offset = 0; + for (const auto& file : files_) { + files[next_offset] = file; + next_offset += file->GetSize(); + } +} + +std::string ConcatenatedVfsFile::GetName() const { + if (files.empty()) + return ""; + if (!name.empty()) + return name; + return files.begin()->second->GetName(); +} + +size_t ConcatenatedVfsFile::GetSize() const { + if (files.empty()) + return 0; + return files.rbegin()->first + files.rbegin()->second->GetSize(); +} + +bool ConcatenatedVfsFile::Resize(size_t new_size) { + return false; +} + +std::shared_ptr ConcatenatedVfsFile::GetContainingDirectory() const { + if (files.empty()) + return nullptr; + return files.begin()->second->GetContainingDirectory(); +} + +bool ConcatenatedVfsFile::IsWritable() const { + return false; +} + +bool ConcatenatedVfsFile::IsReadable() const { + return true; +} + +size_t ConcatenatedVfsFile::Read(u8* data, size_t length, size_t offset) const { + auto entry = files.end(); + for (auto iter = files.begin(); iter != files.end(); ++iter) { + if (iter->first > offset) { + entry = --iter; + break; + } + } + + if (entry == files.end() && offset < files.rbegin()->first + files.rbegin()->second->GetSize()) + --entry; + + if (entry == files.end()) + return 0; + + const auto remaining = entry->second->GetSize() + offset - entry->first; + if (length > remaining) { + return entry->second->Read(data, remaining, offset - entry->first) + + Read(data + remaining, length - remaining, offset + remaining); + } + + return entry->second->Read(data, length, offset - entry->first); +} + +size_t ConcatenatedVfsFile::Write(const u8* data, size_t length, size_t offset) { + return 0; +} + +bool ConcatenatedVfsFile::Rename(std::string_view name) { + return false; +} +} // namespace FileSys diff --git a/src/core/file_sys/vfs_concat.h b/src/core/file_sys/vfs_concat.h new file mode 100644 index 000000000..d319c5786 --- /dev/null +++ b/src/core/file_sys/vfs_concat.h @@ -0,0 +1,41 @@ +// Copyright 2018 yuzu emulator team +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#pragma once + +#include +#include +#include +#include "core/file_sys/vfs.h" + +namespace FileSys { + +// Wrapper function to allow for more efficient handling of files.size() == 0, 1 cases. +VirtualFile ConcatenateFiles(std::vector files, std::string_view name = ""); + +// Class that wraps multiple vfs files and concatenates them, making reads seamless. Currently +// read-only. +class ConcatenatedVfsFile : public VfsFile { + friend VirtualFile ConcatenateFiles(std::vector files, std::string_view name); + + ConcatenatedVfsFile(std::vector files, std::string_view name); + +public: + std::string GetName() const override; + size_t GetSize() const override; + bool Resize(size_t new_size) override; + std::shared_ptr GetContainingDirectory() const override; + bool IsWritable() const override; + bool IsReadable() const override; + size_t Read(u8* data, size_t length, size_t offset) const override; + size_t Write(const u8* data, size_t length, size_t offset) override; + bool Rename(std::string_view name) override; + +private: + // Maps starting offset to file -- more efficient. + boost::container::flat_map files; + std::string name; +}; + +} // namespace FileSys From 5b4119fa7fae78ac5fd24470b357fc162984f63d Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Thu, 9 Aug 2018 20:47:46 -0400 Subject: [PATCH 05/28] loader: Recognize filename '00' as NCA Needed to avoid mismatch filetype warnings on split NAND NCAs --- src/core/loader/loader.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/loader/loader.cpp b/src/core/loader/loader.cpp index 2f5bfc67c..0e690abb3 100644 --- a/src/core/loader/loader.cpp +++ b/src/core/loader/loader.cpp @@ -46,6 +46,8 @@ FileType IdentifyFile(FileSys::VirtualFile file) { FileType GuessFromFilename(const std::string& name) { if (name == "main") return FileType::DeconstructedRomDirectory; + if (name == "00") + return FileType::NCA; const std::string extension = Common::ToLower(std::string(FileUtil::GetExtensionFromFilename(name))); From 95bb1067c1bd78bcd4c346f0445af9d9217743fc Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Thu, 9 Aug 2018 20:48:41 -0400 Subject: [PATCH 06/28] loader: Join 0* files in directory if filename is 00 i.e. Load the concatenated 00+01 if 01 exists as well. Needed for split NAND NCAs. --- src/core/core.cpp | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/src/core/core.cpp b/src/core/core.cpp index 83d4d742b..05af68f22 100644 --- a/src/core/core.cpp +++ b/src/core/core.cpp @@ -17,6 +17,7 @@ #include "core/hle/service/sm/sm.h" #include "core/loader/loader.h" #include "core/settings.h" +#include "file_sys/vfs_concat.h" #include "file_sys/vfs_real.h" #include "video_core/renderer_base.h" #include "video_core/video_core.h" @@ -88,8 +89,39 @@ System::ResultStatus System::SingleStep() { return RunLoop(false); } +static FileSys::VirtualFile GetGameFileFromPath(const FileSys::VirtualFilesystem& vfs, + const std::string& path) { + // To account for split 00+01+etc files. + std::string dir_name; + std::string filename; + Common::SplitPath(path, &dir_name, &filename, nullptr); + if (filename == "00") { + const auto dir = vfs->OpenDirectory(dir_name, FileSys::Mode::Read); + std::vector concat; + for (u8 i = 0; i < 0x10; ++i) { + auto next = dir->GetFile(fmt::format("{:02X}", i)); + if (next != nullptr) + concat.push_back(std::move(next)); + else { + next = dir->GetFile(fmt::format("{:02x}", i)); + if (next != nullptr) + concat.push_back(std::move(next)); + else + break; + } + } + + if (concat.empty()) + return nullptr; + + return FileSys::ConcatenateFiles(concat, dir->GetName()); + } + + return vfs->OpenFile(path, FileSys::Mode::Read); +} + System::ResultStatus System::Load(Frontend::EmuWindow& emu_window, const std::string& filepath) { - app_loader = Loader::GetLoader(virtual_filesystem->OpenFile(filepath, FileSys::Mode::Read)); + app_loader = Loader::GetLoader(GetGameFileFromPath(virtual_filesystem, filepath)); if (!app_loader) { LOG_CRITICAL(Core, "Failed to obtain loader for {}!", filepath); From 70a510bd8f2f8d150abe632c94dc9a3eb247e43f Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Thu, 9 Aug 2018 20:49:45 -0400 Subject: [PATCH 07/28] bis_factory: Add partial implementation of BISFactory Creates and stores RegisteredCaches for user and system NAND, as creation of a RegisteredCache is expensive. --- src/core/file_sys/bis_factory.cpp | 24 ++++++++++++++++++++++++ src/core/file_sys/bis_factory.h | 30 ++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 src/core/file_sys/bis_factory.cpp create mode 100644 src/core/file_sys/bis_factory.h diff --git a/src/core/file_sys/bis_factory.cpp b/src/core/file_sys/bis_factory.cpp new file mode 100644 index 000000000..7d0de733b --- /dev/null +++ b/src/core/file_sys/bis_factory.cpp @@ -0,0 +1,24 @@ +// Copyright 2018 yuzu emulator team +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#include "core/file_sys/bis_factory.h" + +namespace FileSys { + +BISFactory::BISFactory(VirtualDir nand_root_) + : nand_root(std::move(nand_root_)), + sysnand_cache(std::make_shared( + nand_root->GetDirectoryRelative("/system/Contents/registered"))), + usrnand_cache(std::make_shared( + nand_root->GetDirectoryRelative("/user/Contents/registered"))) {} + +std::shared_ptr BISFactory::GetSystemNANDContents() const { + return sysnand_cache; +} + +std::shared_ptr BISFactory::GetUserNANDContents() const { + return usrnand_cache; +} + +} // namespace FileSys diff --git a/src/core/file_sys/bis_factory.h b/src/core/file_sys/bis_factory.h new file mode 100644 index 000000000..a970a5e2e --- /dev/null +++ b/src/core/file_sys/bis_factory.h @@ -0,0 +1,30 @@ +// Copyright 2018 yuzu emulator team +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#pragma once + +#include +#include "core/loader/loader.h" +#include "registered_cache.h" + +namespace FileSys { + +/// File system interface to the Built-In Storage +/// This is currently missing accessors to BIS partitions, but seemed like a good place for the NAND +/// registered caches. +class BISFactory { +public: + explicit BISFactory(VirtualDir nand_root); + + std::shared_ptr GetSystemNANDContents() const; + std::shared_ptr GetUserNANDContents() const; + +private: + VirtualDir nand_root; + + std::shared_ptr sysnand_cache; + std::shared_ptr usrnand_cache; +}; + +} // namespace FileSys From c0257cf52f5ed6c09122a539718d3a18900f61f9 Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Thu, 9 Aug 2018 20:50:10 -0400 Subject: [PATCH 08/28] filesystem: Add Open and Register functions for BISFactory --- .../hle/service/filesystem/filesystem.cpp | 19 +++++++++++++++++++ src/core/hle/service/filesystem/filesystem.h | 8 ++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/core/hle/service/filesystem/filesystem.cpp b/src/core/hle/service/filesystem/filesystem.cpp index 5e416cde2..da658cbe6 100644 --- a/src/core/hle/service/filesystem/filesystem.cpp +++ b/src/core/hle/service/filesystem/filesystem.cpp @@ -226,6 +226,7 @@ ResultVal VfsDirectoryServiceWrapper::GetEntryType( static std::unique_ptr romfs_factory; static std::unique_ptr save_data_factory; static std::unique_ptr sdmc_factory; +static std::unique_ptr bis_factory; ResultCode RegisterRomFS(std::unique_ptr&& factory) { ASSERT_MSG(romfs_factory == nullptr, "Tried to register a second RomFS"); @@ -248,6 +249,13 @@ ResultCode RegisterSDMC(std::unique_ptr&& factory) { return RESULT_SUCCESS; } +ResultCode RegisterBIS(std::unique_ptr&& factory) { + ASSERT_MSG(bis_factory == nullptr, "Tried to register a second BIS"); + bis_factory = std::move(factory); + LOG_DEBUG(Service_FS, "Registred BIS"); + return RESULT_SUCCESS; +} + ResultVal OpenRomFS(u64 title_id) { LOG_TRACE(Service_FS, "Opening RomFS for title_id={:016X}", title_id); @@ -281,6 +289,14 @@ ResultVal OpenSDMC() { return sdmc_factory->Open(); } +std::shared_ptr GetSystemNANDContents() { + return bis_factory->GetSystemNANDContents(); +} + +std::shared_ptr GetUserNANDContents() { + return bis_factory->GetUserNANDContents(); +} + void RegisterFileSystems(const FileSys::VirtualFilesystem& vfs) { romfs_factory = nullptr; save_data_factory = nullptr; @@ -291,6 +307,9 @@ void RegisterFileSystems(const FileSys::VirtualFilesystem& vfs) { auto sd_directory = vfs->OpenDirectory(FileUtil::GetUserPath(FileUtil::UserPath::SDMCDir), FileSys::Mode::ReadWrite); + if (bis_factory == nullptr) + bis_factory = std::make_unique(nand_directory); + auto savedata = std::make_unique(std::move(nand_directory)); save_data_factory = std::move(savedata); diff --git a/src/core/hle/service/filesystem/filesystem.h b/src/core/hle/service/filesystem/filesystem.h index 462c13f20..1d6f922dd 100644 --- a/src/core/hle/service/filesystem/filesystem.h +++ b/src/core/hle/service/filesystem/filesystem.h @@ -6,6 +6,7 @@ #include #include "common/common_types.h" +#include "core/file_sys/bis_factory.h" #include "core/file_sys/directory.h" #include "core/file_sys/mode.h" #include "core/file_sys/romfs_factory.h" @@ -24,16 +25,15 @@ namespace FileSystem { ResultCode RegisterRomFS(std::unique_ptr&& factory); ResultCode RegisterSaveData(std::unique_ptr&& factory); ResultCode RegisterSDMC(std::unique_ptr&& factory); +ResultCode RegisterBIS(std::unique_ptr&& factory); -// TODO(DarkLordZach): BIS Filesystem -// ResultCode RegisterBIS(std::unique_ptr&& factory); ResultVal OpenRomFS(u64 title_id); ResultVal OpenSaveData(FileSys::SaveDataSpaceId space, FileSys::SaveDataDescriptor save_struct); ResultVal OpenSDMC(); -// TODO(DarkLordZach): BIS Filesystem -// ResultVal> OpenBIS(); +std::shared_ptr GetSystemNANDContents(); +std::shared_ptr GetUserNANDContents(); /// Registers all Filesystem services with the specified service manager. void InstallInterfaces(SM::ServiceManager& service_manager, const FileSys::VirtualFilesystem& vfs); From 9b0e3556edf9a77d8d782f56e064935e3e455242 Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Thu, 9 Aug 2018 20:50:50 -0400 Subject: [PATCH 09/28] vfs_real: Add CreateFullPath to CreateFile Fixes bugs with calling CreateFile when the immediate directory does not exist. --- src/core/file_sys/vfs_real.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/core/file_sys/vfs_real.cpp b/src/core/file_sys/vfs_real.cpp index 1b5919737..fa682153c 100644 --- a/src/core/file_sys/vfs_real.cpp +++ b/src/core/file_sys/vfs_real.cpp @@ -83,7 +83,10 @@ VirtualFile RealVfsFilesystem::OpenFile(std::string_view path_, Mode perms) { VirtualFile RealVfsFilesystem::CreateFile(std::string_view path_, Mode perms) { const auto path = FileUtil::SanitizePath(path_, FileUtil::DirectorySeparator::PlatformDefault); - if (!FileUtil::Exists(path) && !FileUtil::CreateEmptyFile(path)) + if (!FileUtil::Exists(path) && + !FileUtil::CreateFullPath( + FileUtil::SanitizePath(path, FileUtil::DirectorySeparator::ForwardSlash)) && + !FileUtil::CreateEmptyFile(path)) return nullptr; return OpenFile(path, perms); } @@ -306,14 +309,14 @@ RealVfsDirectory::RealVfsDirectory(RealVfsFilesystem& base_, const std::string& std::shared_ptr RealVfsDirectory::GetFileRelative(std::string_view path) const { const auto full_path = FileUtil::SanitizePath(this->path + DIR_SEP + std::string(path)); - if (!FileUtil::Exists(full_path)) + if (!FileUtil::Exists(full_path) || FileUtil::IsDirectory(full_path)) return nullptr; return base.OpenFile(full_path, perms); } std::shared_ptr RealVfsDirectory::GetDirectoryRelative(std::string_view path) const { const auto full_path = FileUtil::SanitizePath(this->path + DIR_SEP + std::string(path)); - if (!FileUtil::Exists(full_path)) + if (!FileUtil::Exists(full_path) || !FileUtil::IsDirectory(full_path)) return nullptr; return base.OpenDirectory(full_path, perms); } From ab8acce64598e89c31545713180f3334d1f81991 Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Thu, 9 Aug 2018 20:51:14 -0400 Subject: [PATCH 10/28] card_image: Add accessor for all NCAs in XCI --- src/core/file_sys/card_image.cpp | 4 ++++ src/core/file_sys/card_image.h | 1 + 2 files changed, 5 insertions(+) diff --git a/src/core/file_sys/card_image.cpp b/src/core/file_sys/card_image.cpp index a4823353e..093c625ff 100644 --- a/src/core/file_sys/card_image.cpp +++ b/src/core/file_sys/card_image.cpp @@ -93,6 +93,10 @@ VirtualDir XCI::GetLogoPartition() const { return GetPartition(XCIPartition::Logo); } +const std::vector>& XCI::GetNCAs() const { + return ncas; +} + std::shared_ptr XCI::GetNCAByType(NCAContentType type) const { const auto iter = std::find_if(ncas.begin(), ncas.end(), diff --git a/src/core/file_sys/card_image.h b/src/core/file_sys/card_image.h index e089d737c..3514bdf6c 100644 --- a/src/core/file_sys/card_image.h +++ b/src/core/file_sys/card_image.h @@ -68,6 +68,7 @@ public: VirtualDir GetUpdatePartition() const; VirtualDir GetLogoPartition() const; + const std::vector>& GetNCAs() const; std::shared_ptr GetNCAByType(NCAContentType type) const; VirtualFile GetNCAFileByType(NCAContentType type) const; From 9aab7871222ca86bdf817cc6c96956b25aa76674 Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Thu, 9 Aug 2018 20:51:52 -0400 Subject: [PATCH 11/28] file_sys: Add support for parsing NCA metadata (CNMT) --- src/core/CMakeLists.txt | 8 ++ src/core/file_sys/nca_metadata.cpp | 125 +++++++++++++++++++++++++++++ src/core/file_sys/nca_metadata.h | 105 ++++++++++++++++++++++++ 3 files changed, 238 insertions(+) create mode 100644 src/core/file_sys/nca_metadata.cpp create mode 100644 src/core/file_sys/nca_metadata.h diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index cceb1564b..8cf9fb405 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -20,6 +20,8 @@ add_library(core STATIC crypto/key_manager.h crypto/ctr_encryption_layer.cpp crypto/ctr_encryption_layer.h + file_sys/bis_factory.cpp + file_sys/bis_factory.h file_sys/card_image.cpp file_sys/card_image.h file_sys/content_archive.cpp @@ -29,10 +31,14 @@ add_library(core STATIC file_sys/directory.h file_sys/errors.h file_sys/mode.h + file_sys/nca_metadata.cpp + file_sys/nca_metadata.h file_sys/partition_filesystem.cpp file_sys/partition_filesystem.h file_sys/program_metadata.cpp file_sys/program_metadata.h + file_sys/registered_cache.cpp + file_sys/registered_cache.h file_sys/romfs.cpp file_sys/romfs.h file_sys/romfs_factory.cpp @@ -43,6 +49,8 @@ add_library(core STATIC file_sys/sdmc_factory.h file_sys/vfs.cpp file_sys/vfs.h + file_sys/vfs_concat.cpp + file_sys/vfs_concat.h file_sys/vfs_offset.cpp file_sys/vfs_offset.h file_sys/vfs_real.cpp diff --git a/src/core/file_sys/nca_metadata.cpp b/src/core/file_sys/nca_metadata.cpp new file mode 100644 index 000000000..fa06897b7 --- /dev/null +++ b/src/core/file_sys/nca_metadata.cpp @@ -0,0 +1,125 @@ +// Copyright 2018 yuzu emulator team +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#include "common/common_funcs.h" +#include "common/swap.h" +#include "content_archive.h" +#include "core/file_sys/nca_metadata.h" + +namespace FileSys { + +CNMT::CNMT(VirtualFile file_) : file(std::move(file_)), header(std::make_unique()) { + if (file->ReadObject(header.get()) != sizeof(CNMTHeader)) + return; + + // If type is {Application, Update, AOC} has opt-header. + if (static_cast(header->type) >= 0x80 && static_cast(header->type) <= 0x82) { + opt_header = std::make_unique(); + if (file->ReadObject(opt_header.get(), sizeof(CNMTHeader)) != sizeof(OptionalHeader)) { + opt_header = nullptr; + } + } + + for (u16 i = 0; i < header->number_content_entries; ++i) { + auto& next = content_records.emplace_back(ContentRecord{}); + if (file->ReadObject(&next, sizeof(CNMTHeader) + i * sizeof(ContentRecord) + + header->table_offset) != sizeof(ContentRecord)) { + content_records.erase(content_records.end() - 1); + } + } + + for (u16 i = 0; i < header->number_meta_entries; ++i) { + auto& next = meta_records.emplace_back(MetaRecord{}); + if (file->ReadObject(&next, sizeof(CNMTHeader) + i * sizeof(MetaRecord) + + header->table_offset) != sizeof(MetaRecord)) { + meta_records.erase(meta_records.end() - 1); + } + } +} + +CNMT::CNMT(CNMTHeader header, OptionalHeader opt_header, std::vector content_records, + std::vector meta_records) + : file(nullptr), header(std::make_unique(std::move(header))), + opt_header(std::make_unique(std::move(opt_header))), + content_records(std::move(content_records)), meta_records(std::move(meta_records)) {} + +u64 CNMT::GetTitleID() const { + return header->title_id; +} + +u32 CNMT::GetTitleVersion() const { + return header->title_version; +} + +TitleType CNMT::GetType() const { + return header->type; +} + +const std::vector& CNMT::GetContentRecords() const { + return content_records; +} + +const std::vector& CNMT::GetMetaRecords() const { + return meta_records; +} + +bool CNMT::UnionRecords(const CNMT& other) { + bool change = false; + for (const auto& rec : other.content_records) { + const auto iter = std::find_if( + content_records.begin(), content_records.end(), + [rec](const ContentRecord& r) { return r.nca_id == rec.nca_id && r.type == rec.type; }); + if (iter == content_records.end()) { + content_records.emplace_back(rec); + ++header->number_content_entries; + change = true; + } + } + for (const auto& rec : other.meta_records) { + const auto iter = + std::find_if(meta_records.begin(), meta_records.end(), [rec](const MetaRecord& r) { + return r.title_id == rec.title_id && r.title_version == rec.title_version && + r.type == rec.type; + }); + if (iter == meta_records.end()) { + meta_records.emplace_back(rec); + ++header->number_meta_entries; + change = true; + } + } + return change; +} + +std::vector CNMT::Serialize() const { + if (header == nullptr) + return {}; + std::vector out(sizeof(CNMTHeader)); + out.reserve(0x100); // Avoid resizing -- average size. + memcpy(out.data(), header.get(), sizeof(CNMTHeader)); + if (opt_header != nullptr) { + out.resize(out.size() + sizeof(OptionalHeader)); + memcpy(out.data() + sizeof(CNMTHeader), opt_header.get(), sizeof(OptionalHeader)); + } + + auto offset = header->table_offset; + + const auto dead_zone = offset + sizeof(CNMTHeader) - out.size(); + if (dead_zone > 0) + out.resize(offset + sizeof(CNMTHeader)); + + for (const auto& rec : content_records) { + out.resize(out.size() + sizeof(ContentRecord)); + memcpy(out.data() + offset + sizeof(CNMTHeader), &rec, sizeof(ContentRecord)); + offset += sizeof(ContentRecord); + } + + for (const auto& rec : meta_records) { + out.resize(out.size() + sizeof(MetaRecord)); + memcpy(out.data() + offset + sizeof(CNMTHeader), &rec, sizeof(MetaRecord)); + offset += sizeof(MetaRecord); + } + + return out; +} +} // namespace FileSys diff --git a/src/core/file_sys/nca_metadata.h b/src/core/file_sys/nca_metadata.h new file mode 100644 index 000000000..7b0725f36 --- /dev/null +++ b/src/core/file_sys/nca_metadata.h @@ -0,0 +1,105 @@ +// Copyright 2018 yuzu emulator team +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#pragma once + +#include +#include "core/file_sys/vfs.h" + +namespace FileSys { +class CNMT; + +struct CNMTHeader; +struct OptionalHeader; + +enum class TitleType : u8 { + SystemProgram = 0x01, + SystemDataArchive = 0x02, + SystemUpdate = 0x03, + FirmwarePackageA = 0x04, + FirmwarePackageB = 0x05, + Application = 0x80, + Update = 0x81, + AOC = 0x82, + DeltaTitle = 0x83, +}; + +enum class ContentRecordType : u8 { + Meta = 0, + Program = 1, + Data = 2, + Control = 3, + Manual = 4, + Legal = 5, + Patch = 6, +}; + +struct ContentRecord { + std::array hash; + std::array nca_id; + std::array size; + ContentRecordType type; + INSERT_PADDING_BYTES(1); +}; +static_assert(sizeof(ContentRecord) == 0x38, "ContentRecord has incorrect size."); + +constexpr ContentRecord EMPTY_META_CONTENT_RECORD{{}, {}, {}, ContentRecordType::Meta, {}}; + +struct MetaRecord { + u64_le title_id; + u32_le title_version; + TitleType type; + u8 install_byte; + INSERT_PADDING_BYTES(2); +}; +static_assert(sizeof(MetaRecord) == 0x10, "MetaRecord has incorrect size."); + +struct OptionalHeader { + u64_le title_id; + u64_le minimum_version; +}; +static_assert(sizeof(OptionalHeader) == 0x10, "OptionalHeader has incorrect size."); + +struct CNMTHeader { + u64_le title_id; + u32_le title_version; + TitleType type; + INSERT_PADDING_BYTES(1); + u16_le table_offset; + u16_le number_content_entries; + u16_le number_meta_entries; + INSERT_PADDING_BYTES(12); +}; +static_assert(sizeof(CNMTHeader) == 0x20, "CNMTHeader has incorrect size."); + +// A class representing the format used by NCA metadata files, typically named {}.cnmt.nca or +// meta0.ncd. These describe which NCA's belong with which titles in the registered cache. +class CNMT { +public: + explicit CNMT(VirtualFile file); + CNMT(CNMTHeader header, OptionalHeader opt_header, std::vector content_records, + std::vector meta_records); + + u64 GetTitleID() const; + u32 GetTitleVersion() const; + TitleType GetType() const; + + const std::vector& GetContentRecords() const; + const std::vector& GetMetaRecords() const; + + bool UnionRecords(const CNMT& other); + std::vector Serialize() const; + +private: + VirtualFile file; + std::unique_ptr header; + std::unique_ptr opt_header; + std::vector content_records; + std::vector meta_records; + + // TODO(DarkLordZach): According to switchbrew, for Patch-type there is additional data + // after the table. This is not documented, unfortunately. +}; + +} // namespace FileSys From a91983b11c9aab00065e258ab94b1b99a1f62201 Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Thu, 9 Aug 2018 20:52:27 -0400 Subject: [PATCH 12/28] file_sys: Add RegisteredCache Manages NAND NCA get and install. --- src/core/file_sys/registered_cache.cpp | 435 +++++++++++++++++++++++++ src/core/file_sys/registered_cache.h | 108 ++++++ 2 files changed, 543 insertions(+) create mode 100644 src/core/file_sys/registered_cache.cpp create mode 100644 src/core/file_sys/registered_cache.h diff --git a/src/core/file_sys/registered_cache.cpp b/src/core/file_sys/registered_cache.cpp new file mode 100644 index 000000000..5440cdefb --- /dev/null +++ b/src/core/file_sys/registered_cache.cpp @@ -0,0 +1,435 @@ +// Copyright 2018 yuzu emulator team +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#include +#include +#include "common/assert.h" +#include "common/hex_util.h" +#include "common/logging/log.h" +#include "core/crypto/encryption_layer.h" +#include "core/file_sys/card_image.h" +#include "core/file_sys/nca_metadata.h" +#include "core/file_sys/registered_cache.h" +#include "core/file_sys/vfs_concat.h" + +namespace FileSys { +std::string RegisteredCacheEntry::DebugInfo() const { + return fmt::format("title_id={:016X}, content_type={:02X}", title_id, static_cast(type)); +} + +bool operator<(const RegisteredCacheEntry& lhs, const RegisteredCacheEntry& rhs) { + return (lhs.title_id < rhs.title_id) || (lhs.title_id == rhs.title_id && lhs.type < rhs.type); +} + +static bool FollowsTwoDigitDirFormat(std::string_view name) { + const static std::regex two_digit_regex( + "000000[0123456789abcdefABCDEF][0123456789abcdefABCDEF]"); + return std::regex_match(name.begin(), name.end(), two_digit_regex); +} + +static bool FollowsNcaIdFormat(std::string_view name) { + const static std::regex nca_id_regex("[0123456789abcdefABCDEF]+.nca"); + return name.size() == 36 && std::regex_match(name.begin(), name.end(), nca_id_regex); +} + +static std::string GetRelativePathFromNcaID(const std::array& nca_id, bool second_hex_upper, + bool within_two_digit) { + if (!within_two_digit) + return fmt::format("/{}.nca", HexArrayToString(nca_id, second_hex_upper)); + + Core::Crypto::SHA256Hash hash{}; + mbedtls_sha256(nca_id.data(), nca_id.size(), hash.data(), 0); + return fmt::format("/000000{:02X}/{}.nca", hash[0], HexArrayToString(nca_id, second_hex_upper)); +} + +static std::string GetCNMTName(TitleType type, u64 title_id) { + constexpr std::array TITLE_TYPE_NAMES{ + "SystemProgram", + "SystemData", + "SystemUpdate", + "BootImagePackage", + "BootImagePackageSafe", + "Application", + "Patch", + "AddOnContent", + "" ///< Currently unknown 'DeltaTitle' + }; + + size_t index = static_cast(type); + if (index >= 0x80) + index -= 0x80; + return fmt::format("{}_{:016x}.cnmt", TITLE_TYPE_NAMES[index], title_id); +} + +static ContentRecordType GetCRTypeFromNCAType(NCAContentType type) { + switch (type) { + case NCAContentType::Program: + // TODO(DarkLordZach): Differentiate between Program and Patch + return ContentRecordType::Program; + case NCAContentType::Meta: + return ContentRecordType::Meta; + case NCAContentType::Control: + return ContentRecordType::Control; + case NCAContentType::Data: + return ContentRecordType::Data; + case NCAContentType::Manual: + // TODO(DarkLordZach): Peek at NCA contents to differentiate Manual and Legal. + return ContentRecordType::Manual; + default: + UNREACHABLE(); + } +} + +VirtualFile RegisteredCache::OpenFileOrDirectoryConcat(const VirtualDir& dir, + std::string_view path) const { + if (dir->GetFileRelative(path) != nullptr) + return dir->GetFileRelative(path); + if (dir->GetDirectoryRelative(path) != nullptr) { + const auto nca_dir = dir->GetDirectoryRelative(path); + VirtualFile file = nullptr; + + const auto files = nca_dir->GetFiles(); + if (files.size() == 1 && files[0]->GetName() == "00") + file = files[0]; + else { + std::vector concat; + for (u8 i = 0; i < 0x10; ++i) { + auto next = nca_dir->GetFile(fmt::format("{:02X}", i)); + if (next != nullptr) + concat.push_back(std::move(next)); + else { + next = nca_dir->GetFile(fmt::format("{:02x}", i)); + if (next != nullptr) + concat.push_back(std::move(next)); + else + break; + } + } + + if (concat.empty()) + return nullptr; + + file = FileSys::ConcatenateFiles(concat); + } + + return file; + } + return nullptr; +} + +VirtualFile RegisteredCache::GetFileAtID(NcaID id) const { + VirtualFile file; + for (u8 i = 0; i < 4; ++i) { + file = OpenFileOrDirectoryConcat( + dir, GetRelativePathFromNcaID(id, (i & 0b10) == 0, (i & 0b01) == 0)); + if (file != nullptr) + return file; + } + return file; +} + +boost::optional RegisteredCache::GetNcaIDFromMetadata(u64 title_id, + ContentRecordType type) const { + if (type == ContentRecordType::Meta && meta_id.find(title_id) != meta_id.end()) + return meta_id.at(title_id); + if (meta.find(title_id) == meta.end()) + return boost::none; + + const auto& cnmt = meta.at(title_id); + + const auto iter = std::find_if(cnmt.GetContentRecords().begin(), cnmt.GetContentRecords().end(), + [type](const ContentRecord& rec) { return rec.type == type; }); + if (iter == cnmt.GetContentRecords().end()) + return boost::none; + + return boost::make_optional(iter->nca_id); +} + +void RegisteredCache::AccumulateFiles(std::vector& ids) const { + for (const auto& d2_dir : dir->GetSubdirectories()) { + if (FollowsNcaIdFormat(d2_dir->GetName())) { + ids.push_back(HexStringToArray<0x10, true>(d2_dir->GetName().substr(0, 0x20))); + continue; + } + + if (!FollowsTwoDigitDirFormat(d2_dir->GetName())) + continue; + + for (const auto& nca_dir : d2_dir->GetSubdirectories()) { + if (!FollowsNcaIdFormat(nca_dir->GetName())) + continue; + + ids.push_back(HexStringToArray<0x10, true>(nca_dir->GetName().substr(0, 0x20))); + } + + for (const auto& nca_file : d2_dir->GetFiles()) { + if (!FollowsNcaIdFormat(nca_file->GetName())) + continue; + + ids.push_back(HexStringToArray<0x10, true>(nca_file->GetName().substr(0, 0x20))); + } + } + + for (const auto& d2_file : dir->GetFiles()) { + if (FollowsNcaIdFormat(d2_file->GetName())) + ids.push_back(HexStringToArray<0x10, true>(d2_file->GetName().substr(0, 0x20))); + } +} + +void RegisteredCache::ProcessFiles(const std::vector& ids) { + for (const auto& id : ids) { + const auto file = GetFileAtID(id); + + if (file == nullptr) + continue; + const auto nca = std::make_shared(parser(file, id)); + if (nca->GetStatus() != Loader::ResultStatus::Success || + nca->GetType() != NCAContentType::Meta) + continue; + + const auto section0 = nca->GetSubdirectories()[0]; + + for (const auto& file : section0->GetFiles()) { + if (file->GetExtension() != "cnmt") + continue; + + meta.insert_or_assign(nca->GetTitleId(), CNMT(file)); + meta_id.insert_or_assign(nca->GetTitleId(), id); + break; + } + } +} + +void RegisteredCache::AccumulateYuzuMeta() { + const auto dir = this->dir->GetSubdirectory("yuzu_meta"); + if (dir == nullptr) + return; + + for (const auto& file : dir->GetFiles()) { + if (file->GetExtension() != "cnmt") + continue; + + CNMT cnmt(file); + yuzu_meta.insert_or_assign(cnmt.GetTitleID(), std::move(cnmt)); + } +} + +void RegisteredCache::Refresh() { + if (dir == nullptr) + return; + std::vector ids; + AccumulateFiles(ids); + ProcessFiles(ids); + AccumulateYuzuMeta(); +} + +RegisteredCache::RegisteredCache(VirtualDir dir_, RegisteredCacheParsingFunction parsing_function) + : dir(std::move(dir_)), parser(std::move(parsing_function)) { + Refresh(); +} + +bool RegisteredCache::HasEntry(u64 title_id, ContentRecordType type) const { + return GetEntryRaw(title_id, type) != nullptr; +} + +bool RegisteredCache::HasEntry(RegisteredCacheEntry entry) const { + return GetEntryRaw(entry) != nullptr; +} + +VirtualFile RegisteredCache::GetEntryRaw(u64 title_id, ContentRecordType type) const { + const auto id = GetNcaIDFromMetadata(title_id, type); + if (id == boost::none) + return nullptr; + + return parser(GetFileAtID(id.get()), id.get()); +} + +VirtualFile RegisteredCache::GetEntryRaw(RegisteredCacheEntry entry) const { + return GetEntryRaw(entry.title_id, entry.type); +} + +std::shared_ptr RegisteredCache::GetEntry(u64 title_id, ContentRecordType type) const { + const auto raw = GetEntryRaw(title_id, type); + if (raw == nullptr) + return nullptr; + return std::make_shared(raw); +} + +std::shared_ptr RegisteredCache::GetEntry(RegisteredCacheEntry entry) const { + return GetEntry(entry.title_id, entry.type); +} + +template +void RegisteredCache::IterateAllMetadata( + std::vector& out, std::function proc, + std::function filter) const { + for (const auto& kv : meta) { + const auto& cnmt = kv.second; + if (filter(cnmt, EMPTY_META_CONTENT_RECORD)) + out.push_back(proc(cnmt, EMPTY_META_CONTENT_RECORD)); + for (const auto& rec : cnmt.GetContentRecords()) { + if (GetFileAtID(rec.nca_id) != nullptr && filter(cnmt, rec)) { + out.push_back(proc(cnmt, rec)); + } + } + } + for (const auto& kv : yuzu_meta) { + const auto& cnmt = kv.second; + for (const auto& rec : cnmt.GetContentRecords()) { + if (GetFileAtID(rec.nca_id) != nullptr && filter(cnmt, rec)) { + out.push_back(proc(cnmt, rec)); + } + } + } +} + +std::vector RegisteredCache::ListEntries() const { + std::vector out; + IterateAllMetadata( + out, + [](const CNMT& c, const ContentRecord& r) { + return RegisteredCacheEntry{c.GetTitleID(), r.type}; + }, + [](const CNMT& c, const ContentRecord& r) { return true; }); + return out; +} + +std::vector RegisteredCache::ListEntriesFilter( + boost::optional title_type, boost::optional record_type, + boost::optional title_id) const { + std::vector out; + IterateAllMetadata( + out, + [](const CNMT& c, const ContentRecord& r) { + return RegisteredCacheEntry{c.GetTitleID(), r.type}; + }, + [&title_type, &record_type, &title_id](const CNMT& c, const ContentRecord& r) { + if (title_type != boost::none && title_type.get() != c.GetType()) + return false; + if (record_type != boost::none && record_type.get() != r.type) + return false; + if (title_id != boost::none && title_id.get() != c.GetTitleID()) + return false; + return true; + }); + return out; +} + +static std::shared_ptr GetNCAFromXCIForID(std::shared_ptr xci, const NcaID& id) { + const auto filename = fmt::format("{}.nca", HexArrayToString(id, false)); + const auto iter = + std::find_if(xci->GetNCAs().begin(), xci->GetNCAs().end(), + [&filename](std::shared_ptr nca) { return nca->GetName() == filename; }); + return iter == xci->GetNCAs().end() ? nullptr : *iter; +} + +bool RegisteredCache::InstallEntry(std::shared_ptr xci) { + const auto& ncas = xci->GetNCAs(); + const auto& meta_iter = std::find_if(ncas.begin(), ncas.end(), [](std::shared_ptr nca) { + return nca->GetType() == NCAContentType::Meta; + }); + + if (meta_iter == ncas.end()) { + LOG_ERROR(Loader, "The XCI you are attempting to install does not have a metadata NCA and " + "is therefore malformed. Double check your encryption keys."); + return false; + } + + // Install Metadata File + const auto meta_id_raw = (*meta_iter)->GetName().substr(0, 32); + const auto meta_id = HexStringToArray<16>(meta_id_raw); + if (!RawInstallNCA(*meta_iter, meta_id)) + return false; + + // Install all the other NCAs + const auto section0 = (*meta_iter)->GetSubdirectories()[0]; + const auto cnmt_file = section0->GetFiles()[0]; + const CNMT cnmt(cnmt_file); + for (const auto& record : cnmt.GetContentRecords()) { + const auto nca = GetNCAFromXCIForID(xci, record.nca_id); + if (nca == nullptr || !RawInstallNCA(nca, record.nca_id)) + return false; + } + + Refresh(); + return true; +} + +bool RegisteredCache::InstallEntry(std::shared_ptr nca, TitleType type) { + CNMTHeader header{ + nca->GetTitleId(), ///< Title ID + 0, ///< Ignore/Default title version + type, ///< Type + {}, ///< Padding + 0x10, ///< Default table offset + 1, ///< 1 Content Entry + 0, ///< No Meta Entries + {}, ///< Padding + }; + OptionalHeader opt_header{0, 0}; + ContentRecord c_rec{{}, {}, {}, GetCRTypeFromNCAType(nca->GetType()), {}}; + const auto& data = nca->GetBaseFile()->ReadBytes(0x100000); + mbedtls_sha256(data.data(), data.size(), c_rec.hash.data(), 0); + memcpy(&c_rec.nca_id, &c_rec.hash, 16); + const CNMT new_cnmt(header, opt_header, {c_rec}, {}); + return RawInstallYuzuMeta(new_cnmt) && RawInstallNCA(nca, c_rec.nca_id); +} + +bool RegisteredCache::RawInstallNCA(std::shared_ptr nca, boost::optional override_id) { + const auto in = nca->GetBaseFile(); + Core::Crypto::SHA256Hash hash{}; + + // Calculate NcaID + // NOTE: Because computing the SHA256 of an entire NCA is quite expensive (especially if the + // game is massive), we're going to cheat and only hash the first MB of the NCA. + // Also, for XCIs the NcaID matters, so if the override id isn't none, use that. + NcaID id{}; + if (override_id == boost::none) { + const auto& data = in->ReadBytes(0x100000); + mbedtls_sha256(data.data(), data.size(), hash.data(), 0); + memcpy(id.data(), hash.data(), 16); + } else { + id = override_id.get(); + } + + std::string path = GetRelativePathFromNcaID(id, false, true); + + if (GetFileAtID(id) != nullptr) { + LOG_WARNING(Loader, "OW Attempt"); + return false; + } + + auto out = dir->CreateFileRelative(path); + if (out == nullptr) + return false; + return VfsRawCopy(in, out); +} + +bool RegisteredCache::RawInstallYuzuMeta(const CNMT& cnmt) { + const auto dir = this->dir->CreateDirectoryRelative("yuzu_meta"); + const auto filename = GetCNMTName(cnmt.GetType(), cnmt.GetTitleID()); + if (dir->GetFile(filename) == nullptr) { + auto out = dir->CreateFile(filename); + const auto buffer = cnmt.Serialize(); + out->Resize(buffer.size()); + out->WriteBytes(buffer); + } else { + auto out = dir->GetFile(filename); + CNMT old_cnmt(out); + // Returns true on change + if (old_cnmt.UnionRecords(cnmt)) { + out->Resize(0); + const auto buffer = old_cnmt.Serialize(); + out->Resize(buffer.size()); + out->WriteBytes(buffer); + } + } + Refresh(); + return std::find_if(yuzu_meta.begin(), yuzu_meta.end(), + [&cnmt](const std::pair& kv) { + return kv.second.GetType() == cnmt.GetType() && + kv.second.GetTitleID() == cnmt.GetTitleID(); + }) != yuzu_meta.end(); +} +} // namespace FileSys diff --git a/src/core/file_sys/registered_cache.h b/src/core/file_sys/registered_cache.h new file mode 100644 index 000000000..ba2e3403f --- /dev/null +++ b/src/core/file_sys/registered_cache.h @@ -0,0 +1,108 @@ +// Copyright 2018 yuzu emulator team +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#pragma once + +#include +#include +#include +#include +#include +#include "common/common_funcs.h" +#include "content_archive.h" +#include "core/file_sys/vfs.h" +#include "nca_metadata.h" + +namespace FileSys { +class XCI; +class CNMT; + +using NcaID = std::array; +using RegisteredCacheParsingFunction = std::function; + +struct RegisteredCacheEntry { + u64 title_id; + ContentRecordType type; + + std::string DebugInfo() const; +}; + +// boost flat_map requires operator< for O(log(n)) lookups. +bool operator<(const RegisteredCacheEntry& lhs, const RegisteredCacheEntry& rhs); + +/* + * A class that catalogues NCAs in the registered directory structure. + * Nintendo's registered format follows this structure: + * + * Root + * | 000000XX <- XX is the ____ two digits of the NcaID + * | .nca <- hash is the NcaID (first half of SHA256 over entire file) (folder) + * | 00 + * | 01 <- Actual content split along 4GB boundaries. (optional) + * + * (This impl also supports substituting the nca dir for an nca file, as that's more convenient when + * 4GB splitting can be ignored.) + */ +class RegisteredCache { +public: + // Parsing function defines the conversion from raw file to NCA. If there are other steps + // besides creating the NCA from the file (e.g. NAX0 on SD Card), that should go in a custom + // parsing function. + RegisteredCache(VirtualDir dir, + RegisteredCacheParsingFunction parsing_function = + [](const VirtualFile& file, const NcaID& id) { return file; }); + + void Refresh(); + + bool HasEntry(u64 title_id, ContentRecordType type) const; + bool HasEntry(RegisteredCacheEntry entry) const; + + VirtualFile GetEntryRaw(u64 title_id, ContentRecordType type) const; + VirtualFile GetEntryRaw(RegisteredCacheEntry entry) const; + + std::shared_ptr GetEntry(u64 title_id, ContentRecordType type) const; + std::shared_ptr GetEntry(RegisteredCacheEntry entry) const; + + std::vector ListEntries() const; + // If a parameter is not boost::none, it will be filtered for from all entries. + std::vector ListEntriesFilter( + boost::optional title_type = boost::none, + boost::optional record_type = boost::none, + boost::optional title_id = boost::none) const; + + // Raw copies all the ncas from the xci to the csache. Does some quick checks to make sure there + // is a meta NCA and all of them are accessible. + bool InstallEntry(std::shared_ptr xci); + + // Due to the fact that we must use Meta-type NCAs to determine the existance of files, this + // poses quite a challenge. Instead of creating a new meta NCA for this file, yuzu will create a + // dir inside the NAND called 'yuzu_meta' and store the raw CNMT there. + // TODO(DarkLordZach): Author real meta-type NCAs and install those. + bool InstallEntry(std::shared_ptr nca, TitleType type); + +private: + template + void IterateAllMetadata(std::vector& out, + std::function proc, + std::function filter) const; + void AccumulateFiles(std::vector& ids) const; + void ProcessFiles(const std::vector& ids); + void AccumulateYuzuMeta(); + boost::optional GetNcaIDFromMetadata(u64 title_id, ContentRecordType type) const; + VirtualFile GetFileAtID(NcaID id) const; + VirtualFile OpenFileOrDirectoryConcat(const VirtualDir& dir, std::string_view path) const; + bool RawInstallNCA(std::shared_ptr nca, boost::optional override_id = boost::none); + bool RawInstallYuzuMeta(const CNMT& cnmt); + + VirtualDir dir; + RegisteredCacheParsingFunction parser; + // maps tid -> NcaID of meta + boost::container::flat_map meta_id; + // maps tid -> meta + boost::container::flat_map meta; + // maps tid -> meta for CNMT in yuzu_meta + boost::container::flat_map yuzu_meta; +}; + +} // namespace FileSys From b67e751ccb16a59293c5c3d3ed9f7d3237c9efa4 Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Thu, 9 Aug 2018 20:53:30 -0400 Subject: [PATCH 13/28] game_list: Modify game list to scan installed titles --- src/yuzu/game_list.cpp | 45 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/src/yuzu/game_list.cpp b/src/yuzu/game_list.cpp index 85cb12594..64c72035e 100644 --- a/src/yuzu/game_list.cpp +++ b/src/yuzu/game_list.cpp @@ -2,6 +2,7 @@ // Licensed under GPLv2 or any later version // Refer to the license.txt file included. +#include #include #include #include @@ -403,6 +404,50 @@ void GameList::RefreshGameDirectory() { } void GameListWorker::AddFstEntriesToGameList(const std::string& dir_path, unsigned int recursion) { + const auto usernand = Service::FileSystem::GetUserNANDContents(); + const auto installed_games = usernand->ListEntriesFilter(FileSys::TitleType::Application, + FileSys::ContentRecordType::Program); + + for (const auto& game : installed_games) { + const auto& file = usernand->GetEntryRaw(game); + std::unique_ptr loader = Loader::GetLoader(file); + if (!loader) + continue; + + std::vector icon; + std::string name; + u64 program_id; + loader->ReadProgramId(program_id); + + const auto& control = + usernand->GetEntry(game.title_id, FileSys::ContentRecordType::Control); + if (control != nullptr) { + const auto control_dir = FileSys::ExtractRomFS(control->GetRomFS()); + + const auto nacp_file = control_dir->GetFile("control.nacp"); + FileSys::NACP nacp(nacp_file); + name = nacp.GetApplicationName(); + + FileSys::VirtualFile icon_file = nullptr; + for (const auto& language : FileSys::LANGUAGE_NAMES) { + icon_file = control_dir->GetFile("icon_" + std::string(language) + ".dat"); + if (icon_file != nullptr) { + icon = icon_file->ReadAllBytes(); + break; + } + } + } + emit EntryReady({ + new GameListItemPath( + FormatGameName(file->GetFullPath()), icon, QString::fromStdString(name), + QString::fromStdString(Loader::GetFileTypeString(loader->GetFileType())), + program_id), + new GameListItem( + QString::fromStdString(Loader::GetFileTypeString(loader->GetFileType()))), + new GameListItemSize(file->GetSize()), + }); + } + boost::container::flat_map> nca_control_map; const auto nca_control_callback = From bfb945c2431ca1ccf1c5c43d4e3c7eaedf0bed31 Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Thu, 9 Aug 2018 21:33:13 -0400 Subject: [PATCH 14/28] qt: Add 'Install to NAND' option to menu Prompts for title type on NCA files. --- src/core/core.cpp | 1 + src/core/file_sys/registered_cache.cpp | 2 +- src/yuzu/main.cpp | 89 ++++++++++++++++++++++++++ src/yuzu/main.h | 1 + src/yuzu/main.ui | 7 ++ 5 files changed, 99 insertions(+), 1 deletion(-) diff --git a/src/core/core.cpp b/src/core/core.cpp index 05af68f22..28038ff6f 100644 --- a/src/core/core.cpp +++ b/src/core/core.cpp @@ -5,6 +5,7 @@ #include #include #include "common/logging/log.h" +#include "common/string_util.h" #include "core/core.h" #include "core/core_timing.h" #include "core/gdbstub/gdbstub.h" diff --git a/src/core/file_sys/registered_cache.cpp b/src/core/file_sys/registered_cache.cpp index 5440cdefb..766fef254 100644 --- a/src/core/file_sys/registered_cache.cpp +++ b/src/core/file_sys/registered_cache.cpp @@ -427,7 +427,7 @@ bool RegisteredCache::RawInstallYuzuMeta(const CNMT& cnmt) { } Refresh(); return std::find_if(yuzu_meta.begin(), yuzu_meta.end(), - [&cnmt](const std::pair& kv) { + [&cnmt](const std::pair& kv) { return kv.second.GetType() == cnmt.GetType() && kv.second.GetTitleID() == cnmt.GetTitleID(); }) != yuzu_meta.end(); diff --git a/src/yuzu/main.cpp b/src/yuzu/main.cpp index 94fb8ae6a..c48191486 100644 --- a/src/yuzu/main.cpp +++ b/src/yuzu/main.cpp @@ -24,6 +24,7 @@ #include "common/string_util.h" #include "core/core.h" #include "core/crypto/key_manager.h" +#include "core/file_sys/card_image.h" #include "core/file_sys/vfs_real.h" #include "core/gdbstub/gdbstub.h" #include "core/loader/loader.h" @@ -114,6 +115,9 @@ GMainWindow::GMainWindow() .arg(Common::g_build_name, Common::g_scm_branch, Common::g_scm_desc)); show(); + // Necessary to load titles from nand in gamelist. + Service::FileSystem::RegisterBIS(std::make_unique(vfs->OpenDirectory( + FileUtil::GetUserPath(FileUtil::UserPath::NANDDir), FileSys::Mode::ReadWrite))); game_list->PopulateAsync(UISettings::values.gamedir, UISettings::values.gamedir_deepscan); // Show one-time "callout" messages to the user @@ -309,6 +313,8 @@ void GMainWindow::ConnectMenuEvents() { // File connect(ui.action_Load_File, &QAction::triggered, this, &GMainWindow::OnMenuLoadFile); connect(ui.action_Load_Folder, &QAction::triggered, this, &GMainWindow::OnMenuLoadFolder); + connect(ui.action_Install_File_NAND, &QAction::triggered, this, + &GMainWindow::OnMenuInstallToNAND); connect(ui.action_Select_Game_List_Root, &QAction::triggered, this, &GMainWindow::OnMenuSelectGameListRoot); connect(ui.action_Exit, &QAction::triggered, this, &QMainWindow::close); @@ -612,6 +618,89 @@ void GMainWindow::OnMenuLoadFolder() { } } +void GMainWindow::OnMenuInstallToNAND() { + const static QString file_filter = + tr("Installable Switch File (*.nca *.xci);;Nintendo Content Archive (*.nca);;NX Cartridge " + "Image (*.xci)"); + QString filename = QFileDialog::getOpenFileName(this, tr("Install File"), + UISettings::values.roms_path, file_filter); + if (!filename.isEmpty()) { + if (filename.endsWith("xci", Qt::CaseInsensitive)) { + const auto xci = std::make_shared( + vfs->OpenFile(filename.toStdString(), FileSys::Mode::Read)); + if (xci->GetStatus() != Loader::ResultStatus::Success) { + QMessageBox::critical( + this, tr("Failed to Install XCI"), + tr("The XCI file you provided is invalid. Please double-check your encryption " + "keys and the file and try again.")); + return; + } + if (!Service::FileSystem::GetUserNANDContents()->InstallEntry(xci)) { + QMessageBox::critical( + this, tr("Failed to Install XCI"), + tr("There was an error while attempting to install the provided XCI file. It " + "could have an incorrect format or be missing a metadata entry. Please " + "double-check your file and try again.")); + } else { + QMessageBox::information(this, tr("Successfully Installed XCI"), + tr("The file was successfully installed.")); + game_list->PopulateAsync(UISettings::values.gamedir, + UISettings::values.gamedir_deepscan); + } + } else { + const auto nca = std::make_shared( + vfs->OpenFile(filename.toStdString(), FileSys::Mode::Read)); + if (nca->GetStatus() != Loader::ResultStatus::Success) { + QMessageBox::critical( + this, tr("Failed to Install NCA"), + tr("The NCA file you provided is invalid. Please double-check your encryption " + "keys and the file and try again.")); + return; + } + + const static QStringList tt_options{"System Application", + "System Archive", + "System Application Update", + "Firmware Package (Type A)", + "Firmware Package (Type B)", + "Game", + "Game Update", + "Game DLC", + "Delta Title"}; + bool ok; + const auto item = QInputDialog::getItem( + this, tr("Select NCA Install Type..."), + tr("Please select the type of title you would like to install this NCA as:\n(In " + "most instances, the default 'Game' is fine.)"), + tt_options, 5, false, &ok); + + auto index = tt_options.indexOf(item); + if (!ok || index == -1) { + QMessageBox::critical(this, tr("Failed to Install NCA"), + tr("The title type you selected for the NCA is invalid.")); + return; + } + + if (index >= 5) + index += 0x80; + + if (!Service::FileSystem::GetUserNANDContents()->InstallEntry( + nca, static_cast(index))) { + QMessageBox::critical(this, tr("Failed to Install NCA"), + tr("There was an error while attempting to install the " + "provided NCA file. An error might have occured creating " + "the metadata file or parsing the NCA. Please " + "double-check your file and try again.")); + } else { + QMessageBox::information(this, tr("Successfully Installed NCA"), + tr("The file was successfully installed.")); + game_list->PopulateAsync(UISettings::values.gamedir, + UISettings::values.gamedir_deepscan); + } + } + } +} + void GMainWindow::OnMenuSelectGameListRoot() { QString dir_path = QFileDialog::getExistingDirectory(this, tr("Select Directory")); if (!dir_path.isEmpty()) { diff --git a/src/yuzu/main.h b/src/yuzu/main.h index 74487c58c..5f4d2ab9a 100644 --- a/src/yuzu/main.h +++ b/src/yuzu/main.h @@ -125,6 +125,7 @@ private slots: void OnGameListOpenSaveFolder(u64 program_id); void OnMenuLoadFile(); void OnMenuLoadFolder(); + void OnMenuInstallToNAND(); /// Called whenever a user selects the "File->Select Game List Root" menu item void OnMenuSelectGameListRoot(); void OnMenuRecentFile(); diff --git a/src/yuzu/main.ui b/src/yuzu/main.ui index 22c4cad08..a3bfb2af3 100644 --- a/src/yuzu/main.ui +++ b/src/yuzu/main.ui @@ -57,6 +57,8 @@ Recent Files + + @@ -102,6 +104,11 @@ + + + Install File to NAND... + + Load File... From 167bfddafadb843236c0fa683cf97eaffaa5ea1a Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Thu, 9 Aug 2018 23:10:32 -0400 Subject: [PATCH 15/28] file_sys: Comply to style guidelines --- src/common/hex_util.h | 2 ++ src/core/file_sys/nca_metadata.cpp | 10 ++++--- src/core/file_sys/nca_metadata.h | 4 +++ src/core/file_sys/registered_cache.cpp | 24 +++++++++------- src/core/file_sys/registered_cache.h | 13 +++++---- src/core/file_sys/vfs_concat.cpp | 8 +++--- src/core/file_sys/vfs_concat.h | 6 ++-- src/yuzu/main.cpp | 40 +++++++++++++------------- 8 files changed, 60 insertions(+), 47 deletions(-) diff --git a/src/common/hex_util.h b/src/common/hex_util.h index f16c58aab..13d586015 100644 --- a/src/common/hex_util.h +++ b/src/common/hex_util.h @@ -5,6 +5,8 @@ #pragma once #include +#include +#include #include #include "common/common_types.h" diff --git a/src/core/file_sys/nca_metadata.cpp b/src/core/file_sys/nca_metadata.cpp index fa06897b7..118a0c287 100644 --- a/src/core/file_sys/nca_metadata.cpp +++ b/src/core/file_sys/nca_metadata.cpp @@ -2,6 +2,7 @@ // Licensed under GPLv2 or any later version // Refer to the license.txt file included. +#include #include "common/common_funcs.h" #include "common/swap.h" #include "content_archive.h" @@ -67,9 +68,10 @@ const std::vector& CNMT::GetMetaRecords() const { bool CNMT::UnionRecords(const CNMT& other) { bool change = false; for (const auto& rec : other.content_records) { - const auto iter = std::find_if( - content_records.begin(), content_records.end(), - [rec](const ContentRecord& r) { return r.nca_id == rec.nca_id && r.type == rec.type; }); + const auto iter = std::find_if(content_records.begin(), content_records.end(), + [&rec](const ContentRecord& r) { + return r.nca_id == rec.nca_id && r.type == rec.type; + }); if (iter == content_records.end()) { content_records.emplace_back(rec); ++header->number_content_entries; @@ -78,7 +80,7 @@ bool CNMT::UnionRecords(const CNMT& other) { } for (const auto& rec : other.meta_records) { const auto iter = - std::find_if(meta_records.begin(), meta_records.end(), [rec](const MetaRecord& r) { + std::find_if(meta_records.begin(), meta_records.end(), [&rec](const MetaRecord& r) { return r.title_id == rec.title_id && r.title_version == rec.title_version && r.type == rec.type; }); diff --git a/src/core/file_sys/nca_metadata.h b/src/core/file_sys/nca_metadata.h index 7b0725f36..6cd919e54 100644 --- a/src/core/file_sys/nca_metadata.h +++ b/src/core/file_sys/nca_metadata.h @@ -4,7 +4,11 @@ #pragma once +#include #include +#include +#include "common/common_types.h" +#include "common/swap.h" #include "core/file_sys/vfs.h" namespace FileSys { diff --git a/src/core/file_sys/registered_cache.cpp b/src/core/file_sys/registered_cache.cpp index 766fef254..3e7706171 100644 --- a/src/core/file_sys/registered_cache.cpp +++ b/src/core/file_sys/registered_cache.cpp @@ -23,13 +23,13 @@ bool operator<(const RegisteredCacheEntry& lhs, const RegisteredCacheEntry& rhs) } static bool FollowsTwoDigitDirFormat(std::string_view name) { - const static std::regex two_digit_regex( + static const std::regex two_digit_regex( "000000[0123456789abcdefABCDEF][0123456789abcdefABCDEF]"); return std::regex_match(name.begin(), name.end(), two_digit_regex); } static bool FollowsNcaIdFormat(std::string_view name) { - const static std::regex nca_id_regex("[0123456789abcdefABCDEF]+.nca"); + static const std::regex nca_id_regex("[0123456789abcdefABCDEF]+.nca"); return name.size() == 36 && std::regex_match(name.begin(), name.end(), nca_id_regex); } @@ -56,7 +56,7 @@ static std::string GetCNMTName(TitleType type, u64 title_id) { "" ///< Currently unknown 'DeltaTitle' }; - size_t index = static_cast(type); + auto index = static_cast(type); if (index >= 0x80) index -= 0x80; return fmt::format("{}_{:016x}.cnmt", TITLE_TYPE_NAMES[index], title_id); @@ -90,15 +90,15 @@ VirtualFile RegisteredCache::OpenFileOrDirectoryConcat(const VirtualDir& dir, VirtualFile file = nullptr; const auto files = nca_dir->GetFiles(); - if (files.size() == 1 && files[0]->GetName() == "00") + if (files.size() == 1 && files[0]->GetName() == "00") { file = files[0]; - else { + } else { std::vector concat; for (u8 i = 0; i < 0x10; ++i) { auto next = nca_dir->GetFile(fmt::format("{:02X}", i)); - if (next != nullptr) + if (next != nullptr) { concat.push_back(std::move(next)); - else { + } else { next = nca_dir->GetFile(fmt::format("{:02x}", i)); if (next != nullptr) concat.push_back(std::move(next)); @@ -146,7 +146,8 @@ boost::optional RegisteredCache::GetNcaIDFromMetadata(u64 title_id, return boost::make_optional(iter->nca_id); } -void RegisteredCache::AccumulateFiles(std::vector& ids) const { +std::vector RegisteredCache::AccumulateFiles() const { + std::vector ids; for (const auto& d2_dir : dir->GetSubdirectories()) { if (FollowsNcaIdFormat(d2_dir->GetName())) { ids.push_back(HexStringToArray<0x10, true>(d2_dir->GetName().substr(0, 0x20))); @@ -175,6 +176,7 @@ void RegisteredCache::AccumulateFiles(std::vector& ids) const { if (FollowsNcaIdFormat(d2_file->GetName())) ids.push_back(HexStringToArray<0x10, true>(d2_file->GetName().substr(0, 0x20))); } + return ids; } void RegisteredCache::ProcessFiles(const std::vector& ids) { @@ -185,8 +187,9 @@ void RegisteredCache::ProcessFiles(const std::vector& ids) { continue; const auto nca = std::make_shared(parser(file, id)); if (nca->GetStatus() != Loader::ResultStatus::Success || - nca->GetType() != NCAContentType::Meta) + nca->GetType() != NCAContentType::Meta) { continue; + } const auto section0 = nca->GetSubdirectories()[0]; @@ -218,8 +221,7 @@ void RegisteredCache::AccumulateYuzuMeta() { void RegisteredCache::Refresh() { if (dir == nullptr) return; - std::vector ids; - AccumulateFiles(ids); + const auto ids = AccumulateFiles(); ProcessFiles(ids); AccumulateYuzuMeta(); } diff --git a/src/core/file_sys/registered_cache.h b/src/core/file_sys/registered_cache.h index ba2e3403f..baaed02dd 100644 --- a/src/core/file_sys/registered_cache.h +++ b/src/core/file_sys/registered_cache.h @@ -5,14 +5,17 @@ #pragma once #include +#include #include #include #include +#include #include #include "common/common_funcs.h" +#include "common/common_types.h" #include "content_archive.h" +#include "core/file_sys/nca_metadata.h" #include "core/file_sys/vfs.h" -#include "nca_metadata.h" namespace FileSys { class XCI; @@ -49,9 +52,9 @@ public: // Parsing function defines the conversion from raw file to NCA. If there are other steps // besides creating the NCA from the file (e.g. NAX0 on SD Card), that should go in a custom // parsing function. - RegisteredCache(VirtualDir dir, - RegisteredCacheParsingFunction parsing_function = - [](const VirtualFile& file, const NcaID& id) { return file; }); + explicit RegisteredCache(VirtualDir dir, + RegisteredCacheParsingFunction parsing_function = + [](const VirtualFile& file, const NcaID& id) { return file; }); void Refresh(); @@ -86,7 +89,7 @@ private: void IterateAllMetadata(std::vector& out, std::function proc, std::function filter) const; - void AccumulateFiles(std::vector& ids) const; + std::vector AccumulateFiles() const; void ProcessFiles(const std::vector& ids); void AccumulateYuzuMeta(); boost::optional GetNcaIDFromMetadata(u64 title_id, ContentRecordType type) const; diff --git a/src/core/file_sys/vfs_concat.cpp b/src/core/file_sys/vfs_concat.cpp index 1d439e0a4..88a9a5259 100644 --- a/src/core/file_sys/vfs_concat.cpp +++ b/src/core/file_sys/vfs_concat.cpp @@ -9,17 +9,17 @@ namespace FileSys { -VirtualFile ConcatenateFiles(std::vector files, std::string_view name) { +VirtualFile ConcatenateFiles(std::vector files, std::string name) { if (files.empty()) return nullptr; if (files.size() == 1) return files[0]; - return std::shared_ptr(new ConcatenatedVfsFile(std::move(files), name)); + return std::shared_ptr(new ConcatenatedVfsFile(std::move(files), std::move(name))); } -ConcatenatedVfsFile::ConcatenatedVfsFile(std::vector files_, std::string_view name) - : name(name) { +ConcatenatedVfsFile::ConcatenatedVfsFile(std::vector files_, std::string name) + : name(std::move(name)) { size_t next_offset = 0; for (const auto& file : files_) { files[next_offset] = file; diff --git a/src/core/file_sys/vfs_concat.h b/src/core/file_sys/vfs_concat.h index d319c5786..686d32515 100644 --- a/src/core/file_sys/vfs_concat.h +++ b/src/core/file_sys/vfs_concat.h @@ -12,14 +12,14 @@ namespace FileSys { // Wrapper function to allow for more efficient handling of files.size() == 0, 1 cases. -VirtualFile ConcatenateFiles(std::vector files, std::string_view name = ""); +VirtualFile ConcatenateFiles(std::vector files, std::string name = ""); // Class that wraps multiple vfs files and concatenates them, making reads seamless. Currently // read-only. class ConcatenatedVfsFile : public VfsFile { - friend VirtualFile ConcatenateFiles(std::vector files, std::string_view name); + friend VirtualFile ConcatenateFiles(std::vector files, std::string name); - ConcatenatedVfsFile(std::vector files, std::string_view name); + ConcatenatedVfsFile(std::vector files, std::string name); public: std::string GetName() const override; diff --git a/src/yuzu/main.cpp b/src/yuzu/main.cpp index c48191486..fd237df43 100644 --- a/src/yuzu/main.cpp +++ b/src/yuzu/main.cpp @@ -619,7 +619,7 @@ void GMainWindow::OnMenuLoadFolder() { } void GMainWindow::OnMenuInstallToNAND() { - const static QString file_filter = + const QString file_filter = tr("Installable Switch File (*.nca *.xci);;Nintendo Content Archive (*.nca);;NX Cartridge " "Image (*.xci)"); QString filename = QFileDialog::getOpenFileName(this, tr("Install File"), @@ -629,36 +629,36 @@ void GMainWindow::OnMenuInstallToNAND() { const auto xci = std::make_shared( vfs->OpenFile(filename.toStdString(), FileSys::Mode::Read)); if (xci->GetStatus() != Loader::ResultStatus::Success) { - QMessageBox::critical( + QMessageBox::warning( this, tr("Failed to Install XCI"), tr("The XCI file you provided is invalid. Please double-check your encryption " "keys and the file and try again.")); return; } - if (!Service::FileSystem::GetUserNANDContents()->InstallEntry(xci)) { - QMessageBox::critical( - this, tr("Failed to Install XCI"), - tr("There was an error while attempting to install the provided XCI file. It " - "could have an incorrect format or be missing a metadata entry. Please " - "double-check your file and try again.")); - } else { + if (Service::FileSystem::GetUserNANDContents()->InstallEntry(xci)) { QMessageBox::information(this, tr("Successfully Installed XCI"), tr("The file was successfully installed.")); game_list->PopulateAsync(UISettings::values.gamedir, UISettings::values.gamedir_deepscan); + } else { + QMessageBox::warning( + this, tr("Failed to Install XCI"), + tr("There was an error while attempting to install the provided XCI file. It " + "could have an incorrect format or be missing a metadata entry. Please " + "double-check your file and try again.")); } } else { const auto nca = std::make_shared( vfs->OpenFile(filename.toStdString(), FileSys::Mode::Read)); if (nca->GetStatus() != Loader::ResultStatus::Success) { - QMessageBox::critical( + QMessageBox::warning( this, tr("Failed to Install NCA"), tr("The NCA file you provided is invalid. Please double-check your encryption " "keys and the file and try again.")); return; } - const static QStringList tt_options{"System Application", + static const QStringList tt_options{"System Application", "System Archive", "System Application Update", "Firmware Package (Type A)", @@ -676,26 +676,26 @@ void GMainWindow::OnMenuInstallToNAND() { auto index = tt_options.indexOf(item); if (!ok || index == -1) { - QMessageBox::critical(this, tr("Failed to Install NCA"), - tr("The title type you selected for the NCA is invalid.")); + QMessageBox::warning(this, tr("Failed to Install NCA"), + tr("The title type you selected for the NCA is invalid.")); return; } if (index >= 5) index += 0x80; - if (!Service::FileSystem::GetUserNANDContents()->InstallEntry( + if (Service::FileSystem::GetUserNANDContents()->InstallEntry( nca, static_cast(index))) { - QMessageBox::critical(this, tr("Failed to Install NCA"), - tr("There was an error while attempting to install the " - "provided NCA file. An error might have occured creating " - "the metadata file or parsing the NCA. Please " - "double-check your file and try again.")); - } else { QMessageBox::information(this, tr("Successfully Installed NCA"), tr("The file was successfully installed.")); game_list->PopulateAsync(UISettings::values.gamedir, UISettings::values.gamedir_deepscan); + } else { + QMessageBox::warning(this, tr("Failed to Install NCA"), + tr("There was an error while attempting to install the " + "provided NCA file. An error might have occured creating " + "the metadata file or parsing the NCA. Please " + "double-check your file and try again.")); } } } From e5504a060d4af12682abbdf674834397d566502a Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Fri, 10 Aug 2018 11:10:44 -0400 Subject: [PATCH 16/28] registered_cache: Fix missing reading from yuzu_meta --- src/core/file_sys/registered_cache.cpp | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/core/file_sys/registered_cache.cpp b/src/core/file_sys/registered_cache.cpp index 3e7706171..665126c1c 100644 --- a/src/core/file_sys/registered_cache.cpp +++ b/src/core/file_sys/registered_cache.cpp @@ -129,14 +129,12 @@ VirtualFile RegisteredCache::GetFileAtID(NcaID id) const { return file; } -boost::optional RegisteredCache::GetNcaIDFromMetadata(u64 title_id, - ContentRecordType type) const { - if (type == ContentRecordType::Meta && meta_id.find(title_id) != meta_id.end()) - return meta_id.at(title_id); - if (meta.find(title_id) == meta.end()) +static boost::optional CheckMapForContentRecord( + const boost::container::flat_map& map, u64 title_id, ContentRecordType type) { + if (map.find(title_id) == map.end()) return boost::none; - const auto& cnmt = meta.at(title_id); + const auto& cnmt = map.at(title_id); const auto iter = std::find_if(cnmt.GetContentRecords().begin(), cnmt.GetContentRecords().end(), [type](const ContentRecord& rec) { return rec.type == type; }); @@ -146,6 +144,17 @@ boost::optional RegisteredCache::GetNcaIDFromMetadata(u64 title_id, return boost::make_optional(iter->nca_id); } +boost::optional RegisteredCache::GetNcaIDFromMetadata(u64 title_id, + ContentRecordType type) const { + if (type == ContentRecordType::Meta && meta_id.find(title_id) != meta_id.end()) + return meta_id.at(title_id); + + const auto res1 = CheckMapForContentRecord(yuzu_meta, title_id, type); + if (res1 != boost::none) + return res1; + return CheckMapForContentRecord(meta, title_id, type); +} + std::vector RegisteredCache::AccumulateFiles() const { std::vector ids; for (const auto& d2_dir : dir->GetSubdirectories()) { @@ -398,7 +407,7 @@ bool RegisteredCache::RawInstallNCA(std::shared_ptr nca, boost::optional Date: Fri, 10 Aug 2018 11:11:33 -0400 Subject: [PATCH 17/28] game_list: Populate control data from installed NAND --- src/yuzu/game_list.cpp | 64 ++++++++++++++++++++++-------------------- src/yuzu/main.cpp | 2 +- 2 files changed, 35 insertions(+), 31 deletions(-) diff --git a/src/yuzu/game_list.cpp b/src/yuzu/game_list.cpp index 64c72035e..73a0aa281 100644 --- a/src/yuzu/game_list.cpp +++ b/src/yuzu/game_list.cpp @@ -403,6 +403,28 @@ void GameList::RefreshGameDirectory() { } } +static void GetMetadataFromControlNCA(const std::shared_ptr& nca, + std::vector& icon, std::string& name) { + const auto control_dir = FileSys::ExtractRomFS(nca->GetRomFS()); + if (control_dir == nullptr) + return; + + const auto nacp_file = control_dir->GetFile("control.nacp"); + if (nacp_file == nullptr) + return; + FileSys::NACP nacp(nacp_file); + name = nacp.GetApplicationName(); + + FileSys::VirtualFile icon_file = nullptr; + for (const auto& language : FileSys::LANGUAGE_NAMES) { + icon_file = control_dir->GetFile("icon_" + std::string(language) + ".dat"); + if (icon_file != nullptr) { + icon = icon_file->ReadAllBytes(); + break; + } + } +} + void GameListWorker::AddFstEntriesToGameList(const std::string& dir_path, unsigned int recursion) { const auto usernand = Service::FileSystem::GetUserNANDContents(); const auto installed_games = usernand->ListEntriesFilter(FileSys::TitleType::Application, @@ -421,22 +443,8 @@ void GameListWorker::AddFstEntriesToGameList(const std::string& dir_path, unsign const auto& control = usernand->GetEntry(game.title_id, FileSys::ContentRecordType::Control); - if (control != nullptr) { - const auto control_dir = FileSys::ExtractRomFS(control->GetRomFS()); - - const auto nacp_file = control_dir->GetFile("control.nacp"); - FileSys::NACP nacp(nacp_file); - name = nacp.GetApplicationName(); - - FileSys::VirtualFile icon_file = nullptr; - for (const auto& language : FileSys::LANGUAGE_NAMES) { - icon_file = control_dir->GetFile("icon_" + std::string(language) + ".dat"); - if (icon_file != nullptr) { - icon = icon_file->ReadAllBytes(); - break; - } - } - } + if (control != nullptr) + GetMetadataFromControlNCA(control, icon, name); emit EntryReady({ new GameListItemPath( FormatGameName(file->GetFullPath()), icon, QString::fromStdString(name), @@ -450,6 +458,15 @@ void GameListWorker::AddFstEntriesToGameList(const std::string& dir_path, unsign boost::container::flat_map> nca_control_map; + const auto control_data = usernand->ListEntriesFilter(FileSys::TitleType::Application, + FileSys::ContentRecordType::Control); + + for (const auto& entry : control_data) { + const auto nca = usernand->GetEntry(entry); + if (nca != nullptr) + nca_control_map.insert_or_assign(entry.title_id, nca); + } + const auto nca_control_callback = [this, &nca_control_map](u64* num_entries_out, const std::string& directory, const std::string& virtual_name) -> bool { @@ -503,20 +520,7 @@ void GameListWorker::AddFstEntriesToGameList(const std::string& dir_path, unsign // Use from metadata pool. if (nca_control_map.find(program_id) != nca_control_map.end()) { const auto nca = nca_control_map[program_id]; - const auto control_dir = FileSys::ExtractRomFS(nca->GetRomFS()); - - const auto nacp_file = control_dir->GetFile("control.nacp"); - FileSys::NACP nacp(nacp_file); - name = nacp.GetApplicationName(); - - FileSys::VirtualFile icon_file = nullptr; - for (const auto& language : FileSys::LANGUAGE_NAMES) { - icon_file = control_dir->GetFile("icon_" + std::string(language) + ".dat"); - if (icon_file != nullptr) { - icon = icon_file->ReadAllBytes(); - break; - } - } + GetMetadataFromControlNCA(nca, icon, name); } } diff --git a/src/yuzu/main.cpp b/src/yuzu/main.cpp index fd237df43..1f5a9bb02 100644 --- a/src/yuzu/main.cpp +++ b/src/yuzu/main.cpp @@ -682,7 +682,7 @@ void GMainWindow::OnMenuInstallToNAND() { } if (index >= 5) - index += 0x80; + index += 0x7B; if (Service::FileSystem::GetUserNANDContents()->InstallEntry( nca, static_cast(index))) { From 3b3c919e204a8fc5d2a4e7f372514f257acbe96e Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Fri, 10 Aug 2018 15:06:37 -0400 Subject: [PATCH 18/28] registration: Take RawCopy function as parameter Instead of defaulting to VfsRawCopy --- src/core/file_sys/registered_cache.cpp | 16 +++++++++------- src/core/file_sys/registered_cache.h | 9 ++++++--- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/core/file_sys/registered_cache.cpp b/src/core/file_sys/registered_cache.cpp index 665126c1c..aaadb7463 100644 --- a/src/core/file_sys/registered_cache.cpp +++ b/src/core/file_sys/registered_cache.cpp @@ -335,7 +335,7 @@ static std::shared_ptr GetNCAFromXCIForID(std::shared_ptr xci, const N return iter == xci->GetNCAs().end() ? nullptr : *iter; } -bool RegisteredCache::InstallEntry(std::shared_ptr xci) { +bool RegisteredCache::InstallEntry(std::shared_ptr xci, const VfsCopyFunction& copy) { const auto& ncas = xci->GetNCAs(); const auto& meta_iter = std::find_if(ncas.begin(), ncas.end(), [](std::shared_ptr nca) { return nca->GetType() == NCAContentType::Meta; @@ -350,7 +350,7 @@ bool RegisteredCache::InstallEntry(std::shared_ptr xci) { // Install Metadata File const auto meta_id_raw = (*meta_iter)->GetName().substr(0, 32); const auto meta_id = HexStringToArray<16>(meta_id_raw); - if (!RawInstallNCA(*meta_iter, meta_id)) + if (!RawInstallNCA(*meta_iter, copy, meta_id)) return false; // Install all the other NCAs @@ -359,7 +359,7 @@ bool RegisteredCache::InstallEntry(std::shared_ptr xci) { const CNMT cnmt(cnmt_file); for (const auto& record : cnmt.GetContentRecords()) { const auto nca = GetNCAFromXCIForID(xci, record.nca_id); - if (nca == nullptr || !RawInstallNCA(nca, record.nca_id)) + if (nca == nullptr || !RawInstallNCA(nca, copy, record.nca_id)) return false; } @@ -367,7 +367,8 @@ bool RegisteredCache::InstallEntry(std::shared_ptr xci) { return true; } -bool RegisteredCache::InstallEntry(std::shared_ptr nca, TitleType type) { +bool RegisteredCache::InstallEntry(std::shared_ptr nca, TitleType type, + const VfsCopyFunction& copy) { CNMTHeader header{ nca->GetTitleId(), ///< Title ID 0, ///< Ignore/Default title version @@ -384,10 +385,11 @@ bool RegisteredCache::InstallEntry(std::shared_ptr nca, TitleType type) { mbedtls_sha256(data.data(), data.size(), c_rec.hash.data(), 0); memcpy(&c_rec.nca_id, &c_rec.hash, 16); const CNMT new_cnmt(header, opt_header, {c_rec}, {}); - return RawInstallYuzuMeta(new_cnmt) && RawInstallNCA(nca, c_rec.nca_id); + return RawInstallYuzuMeta(new_cnmt) && RawInstallNCA(nca, copy, c_rec.nca_id); } -bool RegisteredCache::RawInstallNCA(std::shared_ptr nca, boost::optional override_id) { +bool RegisteredCache::RawInstallNCA(std::shared_ptr nca, const VfsCopyFunction& copy, + boost::optional override_id) { const auto in = nca->GetBaseFile(); Core::Crypto::SHA256Hash hash{}; @@ -414,7 +416,7 @@ bool RegisteredCache::RawInstallNCA(std::shared_ptr nca, boost::optionalCreateFileRelative(path); if (out == nullptr) return false; - return VfsRawCopy(in, out); + return copy(in, out); } bool RegisteredCache::RawInstallYuzuMeta(const CNMT& cnmt) { diff --git a/src/core/file_sys/registered_cache.h b/src/core/file_sys/registered_cache.h index baaed02dd..f2b07eec8 100644 --- a/src/core/file_sys/registered_cache.h +++ b/src/core/file_sys/registered_cache.h @@ -23,6 +23,7 @@ class CNMT; using NcaID = std::array; using RegisteredCacheParsingFunction = std::function; +using VfsCopyFunction = std::function; struct RegisteredCacheEntry { u64 title_id; @@ -76,13 +77,14 @@ public: // Raw copies all the ncas from the xci to the csache. Does some quick checks to make sure there // is a meta NCA and all of them are accessible. - bool InstallEntry(std::shared_ptr xci); + bool InstallEntry(std::shared_ptr xci, const VfsCopyFunction& copy = &VfsRawCopy); // Due to the fact that we must use Meta-type NCAs to determine the existance of files, this // poses quite a challenge. Instead of creating a new meta NCA for this file, yuzu will create a // dir inside the NAND called 'yuzu_meta' and store the raw CNMT there. // TODO(DarkLordZach): Author real meta-type NCAs and install those. - bool InstallEntry(std::shared_ptr nca, TitleType type); + bool InstallEntry(std::shared_ptr nca, TitleType type, + const VfsCopyFunction& copy = &VfsRawCopy); private: template @@ -95,7 +97,8 @@ private: boost::optional GetNcaIDFromMetadata(u64 title_id, ContentRecordType type) const; VirtualFile GetFileAtID(NcaID id) const; VirtualFile OpenFileOrDirectoryConcat(const VirtualDir& dir, std::string_view path) const; - bool RawInstallNCA(std::shared_ptr nca, boost::optional override_id = boost::none); + bool RawInstallNCA(std::shared_ptr nca, const VfsCopyFunction& copy, + boost::optional override_id = boost::none); bool RawInstallYuzuMeta(const CNMT& cnmt); VirtualDir dir; From f78a6e752f70150f930eb66fe735723f3ffe5654 Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Fri, 10 Aug 2018 15:07:06 -0400 Subject: [PATCH 19/28] qt: Use custom RawCopy with progress bar for installs --- src/yuzu/main.cpp | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/src/yuzu/main.cpp b/src/yuzu/main.cpp index 1f5a9bb02..e8254c30f 100644 --- a/src/yuzu/main.cpp +++ b/src/yuzu/main.cpp @@ -624,6 +624,32 @@ void GMainWindow::OnMenuInstallToNAND() { "Image (*.xci)"); QString filename = QFileDialog::getOpenFileName(this, tr("Install File"), UISettings::values.roms_path, file_filter); + + const auto qt_raw_copy = [this](FileSys::VirtualFile src, FileSys::VirtualFile dest) { + if (src == nullptr || dest == nullptr) + return false; + if (!dest->Resize(src->GetSize())) + return false; + + QProgressDialog progress(fmt::format("Installing file \"{}\"...", src->GetName()).c_str(), + "Cancel", 0, src->GetSize() / 0x1000, this); + progress.setWindowModality(Qt::WindowModal); + + std::array buffer{}; + for (size_t i = 0; i < src->GetSize(); i += 0x1000) { + if (progress.wasCanceled()) { + dest->Resize(0); + return false; + } + + progress.setValue(i / 0x1000); + const auto read = src->Read(buffer.data(), buffer.size(), i); + dest->Write(buffer.data(), read, i); + } + + return true; + }; + if (!filename.isEmpty()) { if (filename.endsWith("xci", Qt::CaseInsensitive)) { const auto xci = std::make_shared( @@ -635,7 +661,7 @@ void GMainWindow::OnMenuInstallToNAND() { "keys and the file and try again.")); return; } - if (Service::FileSystem::GetUserNANDContents()->InstallEntry(xci)) { + if (Service::FileSystem::GetUserNANDContents()->InstallEntry(xci, qt_raw_copy)) { QMessageBox::information(this, tr("Successfully Installed XCI"), tr("The file was successfully installed.")); game_list->PopulateAsync(UISettings::values.gamedir, @@ -685,7 +711,7 @@ void GMainWindow::OnMenuInstallToNAND() { index += 0x7B; if (Service::FileSystem::GetUserNANDContents()->InstallEntry( - nca, static_cast(index))) { + nca, static_cast(index), qt_raw_copy)) { QMessageBox::information(this, tr("Successfully Installed NCA"), tr("The file was successfully installed.")); game_list->PopulateAsync(UISettings::values.gamedir, From 62e859c6c7ee3baed499d34e928fce17b8f8be9e Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Fri, 10 Aug 2018 20:47:25 -0400 Subject: [PATCH 20/28] bis_factory: Create NAND dirs if they don't exist --- src/core/file_sys/bis_factory.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/core/file_sys/bis_factory.cpp b/src/core/file_sys/bis_factory.cpp index 7d0de733b..ae4e33800 100644 --- a/src/core/file_sys/bis_factory.cpp +++ b/src/core/file_sys/bis_factory.cpp @@ -6,12 +6,19 @@ namespace FileSys { +static VirtualDir GetOrCreateDirectory(const VirtualDir& dir, std::string_view path) { + const auto res = dir->GetDirectoryRelative(path); + if (res == nullptr) + return dir->CreateDirectoryRelative(path); + return res; +} + BISFactory::BISFactory(VirtualDir nand_root_) : nand_root(std::move(nand_root_)), sysnand_cache(std::make_shared( - nand_root->GetDirectoryRelative("/system/Contents/registered"))), + GetOrCreateDirectory(nand_root, "/system/Contents/registered"))), usrnand_cache(std::make_shared( - nand_root->GetDirectoryRelative("/user/Contents/registered"))) {} + GetOrCreateDirectory(nand_root, "/user/Contents/registered"))) {} std::shared_ptr BISFactory::GetSystemNANDContents() const { return sysnand_cache; From 22bdddd6f01e1976590e6df55fd3bcb29cb8aeef Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Fri, 10 Aug 2018 20:47:51 -0400 Subject: [PATCH 21/28] nca_metadata: Remove unnecessary reference to base file --- src/core/file_sys/nca_metadata.cpp | 4 ++-- src/core/file_sys/nca_metadata.h | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/core/file_sys/nca_metadata.cpp b/src/core/file_sys/nca_metadata.cpp index 118a0c287..38b5eae80 100644 --- a/src/core/file_sys/nca_metadata.cpp +++ b/src/core/file_sys/nca_metadata.cpp @@ -10,7 +10,7 @@ namespace FileSys { -CNMT::CNMT(VirtualFile file_) : file(std::move(file_)), header(std::make_unique()) { +CNMT::CNMT(VirtualFile file) : header(std::make_unique()) { if (file->ReadObject(header.get()) != sizeof(CNMTHeader)) return; @@ -41,7 +41,7 @@ CNMT::CNMT(VirtualFile file_) : file(std::move(file_)), header(std::make_unique< CNMT::CNMT(CNMTHeader header, OptionalHeader opt_header, std::vector content_records, std::vector meta_records) - : file(nullptr), header(std::make_unique(std::move(header))), + : header(std::make_unique(std::move(header))), opt_header(std::make_unique(std::move(opt_header))), content_records(std::move(content_records)), meta_records(std::move(meta_records)) {} diff --git a/src/core/file_sys/nca_metadata.h b/src/core/file_sys/nca_metadata.h index 6cd919e54..c1ca9b061 100644 --- a/src/core/file_sys/nca_metadata.h +++ b/src/core/file_sys/nca_metadata.h @@ -96,7 +96,6 @@ public: std::vector Serialize() const; private: - VirtualFile file; std::unique_ptr header; std::unique_ptr opt_header; std::vector content_records; From 893447b6b0f5068f3cc2111b5f21c3cff68002e2 Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Sat, 11 Aug 2018 15:39:09 -0400 Subject: [PATCH 22/28] registration: Update documentation and style --- src/core/file_sys/nca_metadata.cpp | 63 ++++++++++++++------------ src/core/file_sys/nca_metadata.h | 7 ++- src/core/file_sys/registered_cache.cpp | 23 +++++++--- src/core/file_sys/vfs_concat.cpp | 1 + src/core/file_sys/vfs_real.cpp | 17 +++++-- 5 files changed, 69 insertions(+), 42 deletions(-) diff --git a/src/core/file_sys/nca_metadata.cpp b/src/core/file_sys/nca_metadata.cpp index 38b5eae80..234d70199 100644 --- a/src/core/file_sys/nca_metadata.cpp +++ b/src/core/file_sys/nca_metadata.cpp @@ -4,36 +4,44 @@ #include #include "common/common_funcs.h" +#include "common/logging/log.h" #include "common/swap.h" #include "content_archive.h" #include "core/file_sys/nca_metadata.h" namespace FileSys { -CNMT::CNMT(VirtualFile file) : header(std::make_unique()) { - if (file->ReadObject(header.get()) != sizeof(CNMTHeader)) +bool operator>=(TitleType lhs, TitleType rhs) { + return static_cast(lhs) >= static_cast(rhs); +} + +bool operator<=(TitleType lhs, TitleType rhs) { + return static_cast(lhs) <= static_cast(rhs); +} + +CNMT::CNMT(VirtualFile file) { + if (file->ReadObject(&header) != sizeof(CNMTHeader)) return; // If type is {Application, Update, AOC} has opt-header. - if (static_cast(header->type) >= 0x80 && static_cast(header->type) <= 0x82) { - opt_header = std::make_unique(); - if (file->ReadObject(opt_header.get(), sizeof(CNMTHeader)) != sizeof(OptionalHeader)) { - opt_header = nullptr; + if (header.type >= TitleType::Application && header.type <= TitleType::AOC) { + if (file->ReadObject(&opt_header, sizeof(CNMTHeader)) != sizeof(OptionalHeader)) { + LOG_WARNING(Loader, "Failed to read optional header."); } } - for (u16 i = 0; i < header->number_content_entries; ++i) { + for (u16 i = 0; i < header.number_content_entries; ++i) { auto& next = content_records.emplace_back(ContentRecord{}); if (file->ReadObject(&next, sizeof(CNMTHeader) + i * sizeof(ContentRecord) + - header->table_offset) != sizeof(ContentRecord)) { + header.table_offset) != sizeof(ContentRecord)) { content_records.erase(content_records.end() - 1); } } - for (u16 i = 0; i < header->number_meta_entries; ++i) { + for (u16 i = 0; i < header.number_meta_entries; ++i) { auto& next = meta_records.emplace_back(MetaRecord{}); if (file->ReadObject(&next, sizeof(CNMTHeader) + i * sizeof(MetaRecord) + - header->table_offset) != sizeof(MetaRecord)) { + header.table_offset) != sizeof(MetaRecord)) { meta_records.erase(meta_records.end() - 1); } } @@ -41,20 +49,19 @@ CNMT::CNMT(VirtualFile file) : header(std::make_unique()) { CNMT::CNMT(CNMTHeader header, OptionalHeader opt_header, std::vector content_records, std::vector meta_records) - : header(std::make_unique(std::move(header))), - opt_header(std::make_unique(std::move(opt_header))), + : header(std::move(header)), opt_header(std::move(opt_header)), content_records(std::move(content_records)), meta_records(std::move(meta_records)) {} u64 CNMT::GetTitleID() const { - return header->title_id; + return header.title_id; } u32 CNMT::GetTitleVersion() const { - return header->title_version; + return header.title_version; } TitleType CNMT::GetType() const { - return header->type; + return header.type; } const std::vector& CNMT::GetContentRecords() const { @@ -74,7 +81,7 @@ bool CNMT::UnionRecords(const CNMT& other) { }); if (iter == content_records.end()) { content_records.emplace_back(rec); - ++header->number_content_entries; + ++header.number_content_entries; change = true; } } @@ -86,7 +93,7 @@ bool CNMT::UnionRecords(const CNMT& other) { }); if (iter == meta_records.end()) { meta_records.emplace_back(rec); - ++header->number_meta_entries; + ++header.number_meta_entries; change = true; } } @@ -94,30 +101,30 @@ bool CNMT::UnionRecords(const CNMT& other) { } std::vector CNMT::Serialize() const { - if (header == nullptr) - return {}; - std::vector out(sizeof(CNMTHeader)); - out.reserve(0x100); // Avoid resizing -- average size. - memcpy(out.data(), header.get(), sizeof(CNMTHeader)); - if (opt_header != nullptr) { - out.resize(out.size() + sizeof(OptionalHeader)); - memcpy(out.data() + sizeof(CNMTHeader), opt_header.get(), sizeof(OptionalHeader)); + const bool has_opt_header = + header.type >= TitleType::Application && header.type <= TitleType::AOC; + std::vector out(sizeof(CNMTHeader) + (has_opt_header ? sizeof(OptionalHeader) : 0)); + memcpy(out.data(), &header, sizeof(CNMTHeader)); + + // Optional Header + if (has_opt_header) { + memcpy(out.data() + sizeof(CNMTHeader), &opt_header, sizeof(OptionalHeader)); } - auto offset = header->table_offset; + auto offset = header.table_offset; const auto dead_zone = offset + sizeof(CNMTHeader) - out.size(); if (dead_zone > 0) out.resize(offset + sizeof(CNMTHeader)); + out.resize(out.size() + content_records.size() * sizeof(ContentRecord)); for (const auto& rec : content_records) { - out.resize(out.size() + sizeof(ContentRecord)); memcpy(out.data() + offset + sizeof(CNMTHeader), &rec, sizeof(ContentRecord)); offset += sizeof(ContentRecord); } + out.resize(out.size() + content_records.size() * sizeof(MetaRecord)); for (const auto& rec : meta_records) { - out.resize(out.size() + sizeof(MetaRecord)); memcpy(out.data() + offset + sizeof(CNMTHeader), &rec, sizeof(MetaRecord)); offset += sizeof(MetaRecord); } diff --git a/src/core/file_sys/nca_metadata.h b/src/core/file_sys/nca_metadata.h index c1ca9b061..88e66d4da 100644 --- a/src/core/file_sys/nca_metadata.h +++ b/src/core/file_sys/nca_metadata.h @@ -29,6 +29,9 @@ enum class TitleType : u8 { DeltaTitle = 0x83, }; +bool operator>=(TitleType lhs, TitleType rhs); +bool operator<=(TitleType lhs, TitleType rhs); + enum class ContentRecordType : u8 { Meta = 0, Program = 1, @@ -96,8 +99,8 @@ public: std::vector Serialize() const; private: - std::unique_ptr header; - std::unique_ptr opt_header; + CNMTHeader header; + OptionalHeader opt_header; std::vector content_records; std::vector meta_records; diff --git a/src/core/file_sys/registered_cache.cpp b/src/core/file_sys/registered_cache.cpp index aaadb7463..20fec2391 100644 --- a/src/core/file_sys/registered_cache.cpp +++ b/src/core/file_sys/registered_cache.cpp @@ -23,13 +23,14 @@ bool operator<(const RegisteredCacheEntry& lhs, const RegisteredCacheEntry& rhs) } static bool FollowsTwoDigitDirFormat(std::string_view name) { - static const std::regex two_digit_regex( - "000000[0123456789abcdefABCDEF][0123456789abcdefABCDEF]"); + static const std::regex two_digit_regex("000000[0-9A-F]{2}", std::regex_constants::ECMAScript | + std::regex_constants::icase); return std::regex_match(name.begin(), name.end(), two_digit_regex); } static bool FollowsNcaIdFormat(std::string_view name) { - static const std::regex nca_id_regex("[0123456789abcdefABCDEF]+.nca"); + static const std::regex nca_id_regex("[0-9A-F]{32}.nca", std::regex_constants::ECMAScript | + std::regex_constants::icase); return name.size() == 36 && std::regex_match(name.begin(), name.end(), nca_id_regex); } @@ -57,8 +58,9 @@ static std::string GetCNMTName(TitleType type, u64 title_id) { }; auto index = static_cast(type); - if (index >= 0x80) - index -= 0x80; + // If the index is after the jump in TitleType, subtract it out. + if (index >= static_cast(TitleType::Application)) + index -= static_cast(TitleType::Application); return fmt::format("{}_{:016x}.cnmt", TITLE_TYPE_NAMES[index], title_id); } @@ -120,9 +122,15 @@ VirtualFile RegisteredCache::OpenFileOrDirectoryConcat(const VirtualDir& dir, VirtualFile RegisteredCache::GetFileAtID(NcaID id) const { VirtualFile file; + // Try all four modes of file storage: + // (bit 1 = uppercase/lower, bit 0 = within a two-digit dir) + // 00: /000000**/{:032X}.nca + // 01: /{:032X}.nca + // 10: /000000**/{:032x}.nca + // 11: /{:032x}.nca for (u8 i = 0; i < 4; ++i) { - file = OpenFileOrDirectoryConcat( - dir, GetRelativePathFromNcaID(id, (i & 0b10) == 0, (i & 0b01) == 0)); + const auto path = GetRelativePathFromNcaID(id, (i & 0b10) == 0, (i & 0b01) == 0); + file = OpenFileOrDirectoryConcat(dir, path); if (file != nullptr) return file; } @@ -420,6 +428,7 @@ bool RegisteredCache::RawInstallNCA(std::shared_ptr nca, const VfsCopyFunct } bool RegisteredCache::RawInstallYuzuMeta(const CNMT& cnmt) { + // Reasoning behind this method can be found in the comment for InstallEntry, NCA overload. const auto dir = this->dir->CreateDirectoryRelative("yuzu_meta"); const auto filename = GetCNMTName(cnmt.GetType(), cnmt.GetTitleID()); if (dir->GetFile(filename) == nullptr) { diff --git a/src/core/file_sys/vfs_concat.cpp b/src/core/file_sys/vfs_concat.cpp index 88a9a5259..e6bf586a3 100644 --- a/src/core/file_sys/vfs_concat.cpp +++ b/src/core/file_sys/vfs_concat.cpp @@ -68,6 +68,7 @@ size_t ConcatenatedVfsFile::Read(u8* data, size_t length, size_t offset) const { } } + // Check if the entry should be the last one. The loop above will make it end(). if (entry == files.end() && offset < files.rbegin()->first + files.rbegin()->second->GetSize()) --entry; diff --git a/src/core/file_sys/vfs_real.cpp b/src/core/file_sys/vfs_real.cpp index fa682153c..33ab35fcd 100644 --- a/src/core/file_sys/vfs_real.cpp +++ b/src/core/file_sys/vfs_real.cpp @@ -83,10 +83,12 @@ VirtualFile RealVfsFilesystem::OpenFile(std::string_view path_, Mode perms) { VirtualFile RealVfsFilesystem::CreateFile(std::string_view path_, Mode perms) { const auto path = FileUtil::SanitizePath(path_, FileUtil::DirectorySeparator::PlatformDefault); - if (!FileUtil::Exists(path) && - !FileUtil::CreateFullPath( - FileUtil::SanitizePath(path, FileUtil::DirectorySeparator::ForwardSlash)) && - !FileUtil::CreateEmptyFile(path)) + if (!FileUtil::Exists(path)) + return nullptr; + if (!FileUtil::CreateFullPath( + FileUtil::SanitizePath(path, FileUtil::DirectorySeparator::ForwardSlash))) + return nullptr; + if (!FileUtil::CreateEmptyFile(path)) return nullptr; return OpenFile(path, perms); } @@ -143,7 +145,12 @@ VirtualDir RealVfsFilesystem::OpenDirectory(std::string_view path_, Mode perms) VirtualDir RealVfsFilesystem::CreateDirectory(std::string_view path_, Mode perms) { const auto path = FileUtil::SanitizePath(path_, FileUtil::DirectorySeparator::PlatformDefault); - if (!FileUtil::Exists(path) && !FileUtil::CreateDir(path)) + if (!FileUtil::Exists(path)) + return nullptr; + if (!FileUtil::CreateFullPath( + FileUtil::SanitizePath(path, FileUtil::DirectorySeparator::ForwardSlash))) + return nullptr; + if (!FileUtil::CreateDir(path)) return nullptr; // Cannot use make_shared as RealVfsDirectory constructor is private return std::shared_ptr(new RealVfsDirectory(*this, path, perms)); From 149bda980a67cf8102aa8290334e697af166aac6 Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Sat, 11 Aug 2018 22:44:50 -0400 Subject: [PATCH 23/28] romfs: Remove cyclic shared_ptr leak in romfs code --- src/core/file_sys/romfs.cpp | 8 ++++---- src/core/file_sys/vfs_vector.cpp | 4 ++-- src/core/file_sys/vfs_vector.h | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/core/file_sys/romfs.cpp b/src/core/file_sys/romfs.cpp index ff3ddb29c..e490c8ace 100644 --- a/src/core/file_sys/romfs.cpp +++ b/src/core/file_sys/romfs.cpp @@ -65,7 +65,7 @@ void ProcessFile(VirtualFile file, size_t file_offset, size_t data_offset, u32 t auto entry = GetEntry(file, file_offset + this_file_offset); parent->AddFile(std::make_shared( - file, entry.first.size, entry.first.offset + data_offset, entry.second, parent)); + file, entry.first.size, entry.first.offset + data_offset, entry.second)); if (entry.first.sibling == ROMFS_ENTRY_EMPTY) break; @@ -79,7 +79,7 @@ void ProcessDirectory(VirtualFile file, size_t dir_offset, size_t file_offset, s while (true) { auto entry = GetEntry(file, dir_offset + this_dir_offset); auto current = std::make_shared( - std::vector{}, std::vector{}, parent, entry.second); + std::vector{}, std::vector{}, entry.second); if (entry.first.child_file != ROMFS_ENTRY_EMPTY) { ProcessFile(file, file_offset, data_offset, entry.first.child_file, current); @@ -108,9 +108,9 @@ VirtualDir ExtractRomFS(VirtualFile file) { const u64 file_offset = header.file_meta.offset; const u64 dir_offset = header.directory_meta.offset + 4; - const auto root = + auto root = std::make_shared(std::vector{}, std::vector{}, - file->GetContainingDirectory(), file->GetName()); + file->GetName(), file->GetContainingDirectory()); ProcessDirectory(file, dir_offset, file_offset, header.data_offset, 0, root); diff --git a/src/core/file_sys/vfs_vector.cpp b/src/core/file_sys/vfs_vector.cpp index fda603960..98e7c4598 100644 --- a/src/core/file_sys/vfs_vector.cpp +++ b/src/core/file_sys/vfs_vector.cpp @@ -8,8 +8,8 @@ namespace FileSys { VectorVfsDirectory::VectorVfsDirectory(std::vector files_, - std::vector dirs_, VirtualDir parent_, - std::string name_) + std::vector dirs_, std::string name_, + VirtualDir parent_) : files(std::move(files_)), dirs(std::move(dirs_)), parent(std::move(parent_)), name(std::move(name_)) {} diff --git a/src/core/file_sys/vfs_vector.h b/src/core/file_sys/vfs_vector.h index ba469647b..dc39c9f2f 100644 --- a/src/core/file_sys/vfs_vector.h +++ b/src/core/file_sys/vfs_vector.h @@ -12,8 +12,8 @@ namespace FileSys { // Vector data is supplied upon construction. struct VectorVfsDirectory : public VfsDirectory { explicit VectorVfsDirectory(std::vector files = {}, - std::vector dirs = {}, VirtualDir parent = nullptr, - std::string name = ""); + std::vector dirs = {}, std::string name = "", + VirtualDir parent = nullptr); std::vector> GetFiles() const override; std::vector> GetSubdirectories() const override; From dda8ef11c7a167ab4f32c84c55d9b8977fa92eb9 Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Sat, 11 Aug 2018 22:45:30 -0400 Subject: [PATCH 24/28] control_metadata: Remove unnecessary reference to base file --- src/core/file_sys/control_metadata.cpp | 2 +- src/core/file_sys/control_metadata.h | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/core/file_sys/control_metadata.cpp b/src/core/file_sys/control_metadata.cpp index 3ddc9f162..ae21ad5b9 100644 --- a/src/core/file_sys/control_metadata.cpp +++ b/src/core/file_sys/control_metadata.cpp @@ -16,7 +16,7 @@ std::string LanguageEntry::GetDeveloperName() const { return Common::StringFromFixedZeroTerminatedBuffer(developer_name.data(), 0x100); } -NACP::NACP(VirtualFile file_) : file(std::move(file_)), raw(std::make_unique()) { +NACP::NACP(VirtualFile file) : raw(std::make_unique()) { file->ReadObject(raw.get()); } diff --git a/src/core/file_sys/control_metadata.h b/src/core/file_sys/control_metadata.h index 6582cc240..8c2cc1a2a 100644 --- a/src/core/file_sys/control_metadata.h +++ b/src/core/file_sys/control_metadata.h @@ -81,7 +81,6 @@ public: std::string GetVersionString() const; private: - VirtualFile file; std::unique_ptr raw; }; From 8f06a0f898fcade16f6ba9376cf4b72ff608f2ad Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Sat, 11 Aug 2018 22:47:25 -0400 Subject: [PATCH 25/28] vfs_real: Add CreateFullPath to Create* operations --- src/core/file_sys/vfs_real.cpp | 18 ++++++------------ src/core/file_sys/vfs_real.h | 1 - 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/src/core/file_sys/vfs_real.cpp b/src/core/file_sys/vfs_real.cpp index 33ab35fcd..02cdb039a 100644 --- a/src/core/file_sys/vfs_real.cpp +++ b/src/core/file_sys/vfs_real.cpp @@ -83,12 +83,9 @@ VirtualFile RealVfsFilesystem::OpenFile(std::string_view path_, Mode perms) { VirtualFile RealVfsFilesystem::CreateFile(std::string_view path_, Mode perms) { const auto path = FileUtil::SanitizePath(path_, FileUtil::DirectorySeparator::PlatformDefault); - if (!FileUtil::Exists(path)) - return nullptr; - if (!FileUtil::CreateFullPath( - FileUtil::SanitizePath(path, FileUtil::DirectorySeparator::ForwardSlash))) - return nullptr; - if (!FileUtil::CreateEmptyFile(path)) + const auto path_fwd = FileUtil::SanitizePath(path, FileUtil::DirectorySeparator::ForwardSlash); + if (!FileUtil::Exists(path) && !FileUtil::CreateFullPath(path_fwd) && + !FileUtil::CreateEmptyFile(path)) return nullptr; return OpenFile(path, perms); } @@ -145,12 +142,9 @@ VirtualDir RealVfsFilesystem::OpenDirectory(std::string_view path_, Mode perms) VirtualDir RealVfsFilesystem::CreateDirectory(std::string_view path_, Mode perms) { const auto path = FileUtil::SanitizePath(path_, FileUtil::DirectorySeparator::PlatformDefault); - if (!FileUtil::Exists(path)) - return nullptr; - if (!FileUtil::CreateFullPath( - FileUtil::SanitizePath(path, FileUtil::DirectorySeparator::ForwardSlash))) - return nullptr; - if (!FileUtil::CreateDir(path)) + const auto path_fwd = FileUtil::SanitizePath(path, FileUtil::DirectorySeparator::ForwardSlash); + if (!FileUtil::Exists(path) && !FileUtil::CreateFullPath(path_fwd) && + !FileUtil::CreateEmptyFile(path)) return nullptr; // Cannot use make_shared as RealVfsDirectory constructor is private return std::shared_ptr(new RealVfsDirectory(*this, path, perms)); diff --git a/src/core/file_sys/vfs_real.h b/src/core/file_sys/vfs_real.h index 8a1e79ef6..989803d43 100644 --- a/src/core/file_sys/vfs_real.h +++ b/src/core/file_sys/vfs_real.h @@ -5,7 +5,6 @@ #pragma once #include - #include #include "common/file_util.h" #include "core/file_sys/mode.h" From fdf27bf39022d8a96c0386cc92b6670953471089 Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Sat, 11 Aug 2018 22:48:27 -0400 Subject: [PATCH 26/28] game_list: Split game list scans to multiple functions Avoids unnecessary rebuilds of control data on every layer of recursion in AddFstEntriesToGameList --- src/yuzu/game_list.cpp | 22 +++++++++++++--------- src/yuzu/game_list_p.h | 3 +++ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/yuzu/game_list.cpp b/src/yuzu/game_list.cpp index 73a0aa281..faaeda63d 100644 --- a/src/yuzu/game_list.cpp +++ b/src/yuzu/game_list.cpp @@ -405,6 +405,7 @@ void GameList::RefreshGameDirectory() { static void GetMetadataFromControlNCA(const std::shared_ptr& nca, std::vector& icon, std::string& name) { + const auto control_dir = FileSys::ExtractRomFS(nca->GetRomFS()); if (control_dir == nullptr) return; @@ -425,7 +426,7 @@ static void GetMetadataFromControlNCA(const std::shared_ptr& nca, } } -void GameListWorker::AddFstEntriesToGameList(const std::string& dir_path, unsigned int recursion) { +void GameListWorker::AddInstalledTitlesToGameList() { const auto usernand = Service::FileSystem::GetUserNANDContents(); const auto installed_games = usernand->ListEntriesFilter(FileSys::TitleType::Application, FileSys::ContentRecordType::Program); @@ -456,8 +457,6 @@ void GameListWorker::AddFstEntriesToGameList(const std::string& dir_path, unsign }); } - boost::container::flat_map> nca_control_map; - const auto control_data = usernand->ListEntriesFilter(FileSys::TitleType::Application, FileSys::ContentRecordType::Control); @@ -466,10 +465,11 @@ void GameListWorker::AddFstEntriesToGameList(const std::string& dir_path, unsign if (nca != nullptr) nca_control_map.insert_or_assign(entry.title_id, nca); } +} - const auto nca_control_callback = - [this, &nca_control_map](u64* num_entries_out, const std::string& directory, - const std::string& virtual_name) -> bool { +void GameListWorker::FillControlMap(const std::string& dir_path) { + const auto nca_control_callback = [this](u64* num_entries_out, const std::string& directory, + const std::string& virtual_name) -> bool { std::string physical_name = directory + DIR_SEP + virtual_name; if (stop_processing) @@ -487,10 +487,11 @@ void GameListWorker::AddFstEntriesToGameList(const std::string& dir_path, unsign }; FileUtil::ForeachDirectoryEntry(nullptr, dir_path, nca_control_callback); +} - const auto callback = [this, recursion, - &nca_control_map](u64* num_entries_out, const std::string& directory, - const std::string& virtual_name) -> bool { +void GameListWorker::AddFstEntriesToGameList(const std::string& dir_path, unsigned int recursion) { + const auto callback = [this, recursion](u64* num_entries_out, const std::string& directory, + const std::string& virtual_name) -> bool { std::string physical_name = directory + DIR_SEP + virtual_name; if (stop_processing) @@ -547,7 +548,10 @@ void GameListWorker::AddFstEntriesToGameList(const std::string& dir_path, unsign void GameListWorker::run() { stop_processing = false; watch_list.append(dir_path); + FillControlMap(dir_path.toStdString()); + AddInstalledTitlesToGameList(); AddFstEntriesToGameList(dir_path.toStdString(), deep_scan ? 256 : 0); + nca_control_map.clear(); emit Finished(watch_list); } diff --git a/src/yuzu/game_list_p.h b/src/yuzu/game_list_p.h index 8fe5e8b80..10c2ef075 100644 --- a/src/yuzu/game_list_p.h +++ b/src/yuzu/game_list_p.h @@ -163,10 +163,13 @@ signals: private: FileSys::VirtualFilesystem vfs; + std::map> nca_control_map; QStringList watch_list; QString dir_path; bool deep_scan; std::atomic_bool stop_processing; + void AddInstalledTitlesToGameList(); + void FillControlMap(const std::string& dir_path); void AddFstEntriesToGameList(const std::string& dir_path, unsigned int recursion = 0); }; From 6b76b774007020befdaa8d7475a9a4edd6d0a0a4 Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Sat, 11 Aug 2018 23:01:38 -0400 Subject: [PATCH 27/28] registration: Add support for force overwrite of installed --- src/core/file_sys/registered_cache.cpp | 50 ++++++++++----- src/core/file_sys/registered_cache.h | 20 ++++-- src/yuzu/game_list.cpp | 1 - src/yuzu/main.cpp | 88 +++++++++++++++++--------- 4 files changed, 106 insertions(+), 53 deletions(-) diff --git a/src/core/file_sys/registered_cache.cpp b/src/core/file_sys/registered_cache.cpp index 20fec2391..e916d5610 100644 --- a/src/core/file_sys/registered_cache.cpp +++ b/src/core/file_sys/registered_cache.cpp @@ -60,7 +60,7 @@ static std::string GetCNMTName(TitleType type, u64 title_id) { auto index = static_cast(type); // If the index is after the jump in TitleType, subtract it out. if (index >= static_cast(TitleType::Application)) - index -= static_cast(TitleType::Application); + index -= 0x7B; return fmt::format("{}_{:016x}.cnmt", TITLE_TYPE_NAMES[index], title_id); } @@ -343,7 +343,8 @@ static std::shared_ptr GetNCAFromXCIForID(std::shared_ptr xci, const N return iter == xci->GetNCAs().end() ? nullptr : *iter; } -bool RegisteredCache::InstallEntry(std::shared_ptr xci, const VfsCopyFunction& copy) { +InstallResult RegisteredCache::InstallEntry(std::shared_ptr xci, bool overwrite_if_exists, + const VfsCopyFunction& copy) { const auto& ncas = xci->GetNCAs(); const auto& meta_iter = std::find_if(ncas.begin(), ncas.end(), [](std::shared_ptr nca) { return nca->GetType() == NCAContentType::Meta; @@ -352,14 +353,16 @@ bool RegisteredCache::InstallEntry(std::shared_ptr xci, const VfsCopyFuncti if (meta_iter == ncas.end()) { LOG_ERROR(Loader, "The XCI you are attempting to install does not have a metadata NCA and " "is therefore malformed. Double check your encryption keys."); - return false; + return InstallResult::ErrorMetaFailed; } // Install Metadata File const auto meta_id_raw = (*meta_iter)->GetName().substr(0, 32); const auto meta_id = HexStringToArray<16>(meta_id_raw); - if (!RawInstallNCA(*meta_iter, copy, meta_id)) - return false; + + const auto res = RawInstallNCA(*meta_iter, copy, overwrite_if_exists, meta_id); + if (res != InstallResult::Success) + return res; // Install all the other NCAs const auto section0 = (*meta_iter)->GetSubdirectories()[0]; @@ -367,16 +370,19 @@ bool RegisteredCache::InstallEntry(std::shared_ptr xci, const VfsCopyFuncti const CNMT cnmt(cnmt_file); for (const auto& record : cnmt.GetContentRecords()) { const auto nca = GetNCAFromXCIForID(xci, record.nca_id); - if (nca == nullptr || !RawInstallNCA(nca, copy, record.nca_id)) - return false; + if (nca == nullptr) + return InstallResult::ErrorCopyFailed; + const auto res2 = RawInstallNCA(nca, copy, overwrite_if_exists, record.nca_id); + if (res2 != InstallResult::Success) + return res2; } Refresh(); - return true; + return InstallResult::Success; } -bool RegisteredCache::InstallEntry(std::shared_ptr nca, TitleType type, - const VfsCopyFunction& copy) { +InstallResult RegisteredCache::InstallEntry(std::shared_ptr nca, TitleType type, + bool overwrite_if_exists, const VfsCopyFunction& copy) { CNMTHeader header{ nca->GetTitleId(), ///< Title ID 0, ///< Ignore/Default title version @@ -393,11 +399,14 @@ bool RegisteredCache::InstallEntry(std::shared_ptr nca, TitleType type, mbedtls_sha256(data.data(), data.size(), c_rec.hash.data(), 0); memcpy(&c_rec.nca_id, &c_rec.hash, 16); const CNMT new_cnmt(header, opt_header, {c_rec}, {}); - return RawInstallYuzuMeta(new_cnmt) && RawInstallNCA(nca, copy, c_rec.nca_id); + if (!RawInstallYuzuMeta(new_cnmt)) + return InstallResult::ErrorMetaFailed; + return RawInstallNCA(nca, copy, overwrite_if_exists, c_rec.nca_id); } -bool RegisteredCache::RawInstallNCA(std::shared_ptr nca, const VfsCopyFunction& copy, - boost::optional override_id) { +InstallResult RegisteredCache::RawInstallNCA(std::shared_ptr nca, const VfsCopyFunction& copy, + bool overwrite_if_exists, + boost::optional override_id) { const auto in = nca->GetBaseFile(); Core::Crypto::SHA256Hash hash{}; @@ -416,15 +425,22 @@ bool RegisteredCache::RawInstallNCA(std::shared_ptr nca, const VfsCopyFunct std::string path = GetRelativePathFromNcaID(id, false, true); - if (GetFileAtID(id) != nullptr) { + if (GetFileAtID(id) != nullptr && !overwrite_if_exists) { LOG_WARNING(Loader, "Attempting to overwrite existing NCA. Skipping..."); - return false; + return InstallResult::ErrorAlreadyExists; + } + + if (GetFileAtID(id) != nullptr) { + LOG_WARNING(Loader, "Overwriting existing NCA..."); + VirtualDir c_dir; + { c_dir = dir->GetFileRelative(path)->GetContainingDirectory(); } + c_dir->DeleteFile(FileUtil::GetFilename(path)); } auto out = dir->CreateFileRelative(path); if (out == nullptr) - return false; - return copy(in, out); + return InstallResult::ErrorCopyFailed; + return copy(in, out) ? InstallResult::Success : InstallResult::ErrorCopyFailed; } bool RegisteredCache::RawInstallYuzuMeta(const CNMT& cnmt) { diff --git a/src/core/file_sys/registered_cache.h b/src/core/file_sys/registered_cache.h index f2b07eec8..a7c51a59c 100644 --- a/src/core/file_sys/registered_cache.h +++ b/src/core/file_sys/registered_cache.h @@ -25,6 +25,13 @@ using NcaID = std::array; using RegisteredCacheParsingFunction = std::function; using VfsCopyFunction = std::function; +enum class InstallResult { + Success, + ErrorAlreadyExists, + ErrorCopyFailed, + ErrorMetaFailed, +}; + struct RegisteredCacheEntry { u64 title_id; ContentRecordType type; @@ -77,14 +84,16 @@ public: // Raw copies all the ncas from the xci to the csache. Does some quick checks to make sure there // is a meta NCA and all of them are accessible. - bool InstallEntry(std::shared_ptr xci, const VfsCopyFunction& copy = &VfsRawCopy); + InstallResult InstallEntry(std::shared_ptr xci, bool overwrite_if_exists = false, + const VfsCopyFunction& copy = &VfsRawCopy); // Due to the fact that we must use Meta-type NCAs to determine the existance of files, this // poses quite a challenge. Instead of creating a new meta NCA for this file, yuzu will create a // dir inside the NAND called 'yuzu_meta' and store the raw CNMT there. // TODO(DarkLordZach): Author real meta-type NCAs and install those. - bool InstallEntry(std::shared_ptr nca, TitleType type, - const VfsCopyFunction& copy = &VfsRawCopy); + InstallResult InstallEntry(std::shared_ptr nca, TitleType type, + bool overwrite_if_exists = false, + const VfsCopyFunction& copy = &VfsRawCopy); private: template @@ -97,8 +106,9 @@ private: boost::optional GetNcaIDFromMetadata(u64 title_id, ContentRecordType type) const; VirtualFile GetFileAtID(NcaID id) const; VirtualFile OpenFileOrDirectoryConcat(const VirtualDir& dir, std::string_view path) const; - bool RawInstallNCA(std::shared_ptr nca, const VfsCopyFunction& copy, - boost::optional override_id = boost::none); + InstallResult RawInstallNCA(std::shared_ptr nca, const VfsCopyFunction& copy, + bool overwrite_if_exists, + boost::optional override_id = boost::none); bool RawInstallYuzuMeta(const CNMT& cnmt); VirtualDir dir; diff --git a/src/yuzu/game_list.cpp b/src/yuzu/game_list.cpp index faaeda63d..f867118d9 100644 --- a/src/yuzu/game_list.cpp +++ b/src/yuzu/game_list.cpp @@ -405,7 +405,6 @@ void GameList::RefreshGameDirectory() { static void GetMetadataFromControlNCA(const std::shared_ptr& nca, std::vector& icon, std::string& name) { - const auto control_dir = FileSys::ExtractRomFS(nca->GetRomFS()); if (control_dir == nullptr) return; diff --git a/src/yuzu/main.cpp b/src/yuzu/main.cpp index e8254c30f..b5f97f332 100644 --- a/src/yuzu/main.cpp +++ b/src/yuzu/main.cpp @@ -650,37 +650,59 @@ void GMainWindow::OnMenuInstallToNAND() { return true; }; + const auto success = [this]() { + QMessageBox::information(this, tr("Successfully Installed"), + tr("The file was successfully installed.")); + game_list->PopulateAsync(UISettings::values.gamedir, UISettings::values.gamedir_deepscan); + }; + + const auto failed = [this]() { + QMessageBox::warning( + this, tr("Failed to Install"), + tr("There was an error while attempting to install the provided file. It " + "could have an incorrect format or be missing metadata. Please " + "double-check your file and try again.")); + }; + + const auto overwrite = [this]() { + return QMessageBox::question(this, "Failed to Install", + "The file you are attempting to install already exists " + "in the cache. Would you like to overwrite it?") == + QMessageBox::Yes; + }; + if (!filename.isEmpty()) { if (filename.endsWith("xci", Qt::CaseInsensitive)) { const auto xci = std::make_shared( vfs->OpenFile(filename.toStdString(), FileSys::Mode::Read)); if (xci->GetStatus() != Loader::ResultStatus::Success) { - QMessageBox::warning( - this, tr("Failed to Install XCI"), - tr("The XCI file you provided is invalid. Please double-check your encryption " - "keys and the file and try again.")); + failed(); return; } - if (Service::FileSystem::GetUserNANDContents()->InstallEntry(xci, qt_raw_copy)) { - QMessageBox::information(this, tr("Successfully Installed XCI"), - tr("The file was successfully installed.")); - game_list->PopulateAsync(UISettings::values.gamedir, - UISettings::values.gamedir_deepscan); + const auto res = + Service::FileSystem::GetUserNANDContents()->InstallEntry(xci, false, qt_raw_copy); + if (res == FileSys::InstallResult::Success) { + success(); } else { - QMessageBox::warning( - this, tr("Failed to Install XCI"), - tr("There was an error while attempting to install the provided XCI file. It " - "could have an incorrect format or be missing a metadata entry. Please " - "double-check your file and try again.")); + if (res == FileSys::InstallResult::ErrorAlreadyExists) { + if (overwrite()) { + const auto res2 = Service::FileSystem::GetUserNANDContents()->InstallEntry( + xci, true, qt_raw_copy); + if (res2 == FileSys::InstallResult::Success) { + success(); + } else { + failed(); + } + } + } else { + failed(); + } } } else { const auto nca = std::make_shared( vfs->OpenFile(filename.toStdString(), FileSys::Mode::Read)); if (nca->GetStatus() != Loader::ResultStatus::Success) { - QMessageBox::warning( - this, tr("Failed to Install NCA"), - tr("The NCA file you provided is invalid. Please double-check your encryption " - "keys and the file and try again.")); + failed(); return; } @@ -702,7 +724,7 @@ void GMainWindow::OnMenuInstallToNAND() { auto index = tt_options.indexOf(item); if (!ok || index == -1) { - QMessageBox::warning(this, tr("Failed to Install NCA"), + QMessageBox::warning(this, tr("Failed to Install"), tr("The title type you selected for the NCA is invalid.")); return; } @@ -710,18 +732,24 @@ void GMainWindow::OnMenuInstallToNAND() { if (index >= 5) index += 0x7B; - if (Service::FileSystem::GetUserNANDContents()->InstallEntry( - nca, static_cast(index), qt_raw_copy)) { - QMessageBox::information(this, tr("Successfully Installed NCA"), - tr("The file was successfully installed.")); - game_list->PopulateAsync(UISettings::values.gamedir, - UISettings::values.gamedir_deepscan); + const auto res = Service::FileSystem::GetUserNANDContents()->InstallEntry( + nca, static_cast(index), false, qt_raw_copy); + if (res == FileSys::InstallResult::Success) { + success(); } else { - QMessageBox::warning(this, tr("Failed to Install NCA"), - tr("There was an error while attempting to install the " - "provided NCA file. An error might have occured creating " - "the metadata file or parsing the NCA. Please " - "double-check your file and try again.")); + if (res == FileSys::InstallResult::ErrorAlreadyExists) { + if (overwrite()) { + const auto res2 = Service::FileSystem::GetUserNANDContents()->InstallEntry( + nca, static_cast(index), true, qt_raw_copy); + if (res2 == FileSys::InstallResult::Success) { + success(); + } else { + failed(); + } + } + } else { + failed(); + } } } } From 35e4a47be0c4ef25f860d51851d2c02c02dff53d Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Sun, 12 Aug 2018 15:55:44 -0400 Subject: [PATCH 28/28] registration: Various style and documentation improvements Fix logic in RealVfsFilesystem Create methods Remove magic numbers Fix regex errors --- src/core/file_sys/nca_metadata.cpp | 11 ++++------- src/core/file_sys/registered_cache.cpp | 13 ++++++++----- src/core/file_sys/vfs_real.cpp | 16 ++++++++++------ 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/src/core/file_sys/nca_metadata.cpp b/src/core/file_sys/nca_metadata.cpp index 234d70199..449244444 100644 --- a/src/core/file_sys/nca_metadata.cpp +++ b/src/core/file_sys/nca_metadata.cpp @@ -103,7 +103,10 @@ bool CNMT::UnionRecords(const CNMT& other) { std::vector CNMT::Serialize() const { const bool has_opt_header = header.type >= TitleType::Application && header.type <= TitleType::AOC; - std::vector out(sizeof(CNMTHeader) + (has_opt_header ? sizeof(OptionalHeader) : 0)); + const auto dead_zone = header.table_offset + sizeof(CNMTHeader); + std::vector out( + std::max(sizeof(CNMTHeader) + (has_opt_header ? sizeof(OptionalHeader) : 0), dead_zone) + + content_records.size() * sizeof(ContentRecord) + meta_records.size() * sizeof(MetaRecord)); memcpy(out.data(), &header, sizeof(CNMTHeader)); // Optional Header @@ -113,17 +116,11 @@ std::vector CNMT::Serialize() const { auto offset = header.table_offset; - const auto dead_zone = offset + sizeof(CNMTHeader) - out.size(); - if (dead_zone > 0) - out.resize(offset + sizeof(CNMTHeader)); - - out.resize(out.size() + content_records.size() * sizeof(ContentRecord)); for (const auto& rec : content_records) { memcpy(out.data() + offset + sizeof(CNMTHeader), &rec, sizeof(ContentRecord)); offset += sizeof(ContentRecord); } - out.resize(out.size() + content_records.size() * sizeof(MetaRecord)); for (const auto& rec : meta_records) { memcpy(out.data() + offset + sizeof(CNMTHeader), &rec, sizeof(MetaRecord)); offset += sizeof(MetaRecord); diff --git a/src/core/file_sys/registered_cache.cpp b/src/core/file_sys/registered_cache.cpp index e916d5610..a5e935f64 100644 --- a/src/core/file_sys/registered_cache.cpp +++ b/src/core/file_sys/registered_cache.cpp @@ -29,8 +29,8 @@ static bool FollowsTwoDigitDirFormat(std::string_view name) { } static bool FollowsNcaIdFormat(std::string_view name) { - static const std::regex nca_id_regex("[0-9A-F]{32}.nca", std::regex_constants::ECMAScript | - std::regex_constants::icase); + static const std::regex nca_id_regex("[0-9A-F]{32}\\.nca", std::regex_constants::ECMAScript | + std::regex_constants::icase); return name.size() == 36 && std::regex_match(name.begin(), name.end(), nca_id_regex); } @@ -59,8 +59,10 @@ static std::string GetCNMTName(TitleType type, u64 title_id) { auto index = static_cast(type); // If the index is after the jump in TitleType, subtract it out. - if (index >= static_cast(TitleType::Application)) - index -= 0x7B; + if (index >= static_cast(TitleType::Application)) { + index -= static_cast(TitleType::Application) - + static_cast(TitleType::FirmwarePackageB); + } return fmt::format("{}_{:016x}.cnmt", TITLE_TYPE_NAMES[index], title_id); } @@ -96,7 +98,8 @@ VirtualFile RegisteredCache::OpenFileOrDirectoryConcat(const VirtualDir& dir, file = files[0]; } else { std::vector concat; - for (u8 i = 0; i < 0x10; ++i) { + // Since the files are a two-digit hex number, max is FF. + for (size_t i = 0; i < 0x100; ++i) { auto next = nca_dir->GetFile(fmt::format("{:02X}", i)); if (next != nullptr) { concat.push_back(std::move(next)); diff --git a/src/core/file_sys/vfs_real.cpp b/src/core/file_sys/vfs_real.cpp index 02cdb039a..0afe515f0 100644 --- a/src/core/file_sys/vfs_real.cpp +++ b/src/core/file_sys/vfs_real.cpp @@ -84,9 +84,11 @@ VirtualFile RealVfsFilesystem::OpenFile(std::string_view path_, Mode perms) { VirtualFile RealVfsFilesystem::CreateFile(std::string_view path_, Mode perms) { const auto path = FileUtil::SanitizePath(path_, FileUtil::DirectorySeparator::PlatformDefault); const auto path_fwd = FileUtil::SanitizePath(path, FileUtil::DirectorySeparator::ForwardSlash); - if (!FileUtil::Exists(path) && !FileUtil::CreateFullPath(path_fwd) && - !FileUtil::CreateEmptyFile(path)) - return nullptr; + if (!FileUtil::Exists(path)) { + FileUtil::CreateFullPath(path_fwd); + if (!FileUtil::CreateEmptyFile(path)) + return nullptr; + } return OpenFile(path, perms); } @@ -143,9 +145,11 @@ VirtualDir RealVfsFilesystem::OpenDirectory(std::string_view path_, Mode perms) VirtualDir RealVfsFilesystem::CreateDirectory(std::string_view path_, Mode perms) { const auto path = FileUtil::SanitizePath(path_, FileUtil::DirectorySeparator::PlatformDefault); const auto path_fwd = FileUtil::SanitizePath(path, FileUtil::DirectorySeparator::ForwardSlash); - if (!FileUtil::Exists(path) && !FileUtil::CreateFullPath(path_fwd) && - !FileUtil::CreateEmptyFile(path)) - return nullptr; + if (!FileUtil::Exists(path)) { + FileUtil::CreateFullPath(path_fwd); + if (!FileUtil::CreateDir(path)) + return nullptr; + } // Cannot use make_shared as RealVfsDirectory constructor is private return std::shared_ptr(new RealVfsDirectory(*this, path, perms)); }