From 906d785c73cb3644e8984fbfcbf5fcb8a1ebbc6f Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Fri, 27 Jul 2018 18:14:03 -0400 Subject: [PATCH] RomFS Extraction --- src/core/CMakeLists.txt | 4 + src/core/file_sys/content_archive.cpp | 19 +--- src/core/file_sys/romfs.cpp | 124 ++++++++++++++++++++++++++ src/core/file_sys/romfs.h | 35 ++++++++ src/core/file_sys/vfs.cpp | 23 +++++ src/core/file_sys/vfs.h | 20 +++++ src/core/file_sys/vfs_offset.cpp | 7 +- src/core/file_sys/vfs_offset.h | 3 +- src/core/file_sys/vfs_real.cpp | 6 ++ src/core/file_sys/vfs_real.h | 3 +- src/core/file_sys/vfs_vector.cpp | 83 +++++++++++++++++ src/core/file_sys/vfs_vector.h | 44 +++++++++ 12 files changed, 351 insertions(+), 20 deletions(-) create mode 100644 src/core/file_sys/romfs.cpp create mode 100644 src/core/file_sys/romfs.h create mode 100644 src/core/file_sys/vfs_vector.cpp create mode 100644 src/core/file_sys/vfs_vector.h diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index b367c2a27..f389897d4 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -23,6 +23,8 @@ add_library(core STATIC file_sys/partition_filesystem.h file_sys/program_metadata.cpp file_sys/program_metadata.h + file_sys/romfs.cpp + file_sys/romfs.h file_sys/romfs_factory.cpp file_sys/romfs_factory.h file_sys/savedata_factory.cpp @@ -35,6 +37,8 @@ add_library(core STATIC file_sys/vfs_offset.h file_sys/vfs_real.cpp file_sys/vfs_real.h + file_sys/vfs_vector.cpp + file_sys/vfs_vector.h frontend/emu_window.cpp frontend/emu_window.h frontend/framebuffer_layout.cpp diff --git a/src/core/file_sys/content_archive.cpp b/src/core/file_sys/content_archive.cpp index d6b20c047..61cb0bbe3 100644 --- a/src/core/file_sys/content_archive.cpp +++ b/src/core/file_sys/content_archive.cpp @@ -9,6 +9,7 @@ #include "core/file_sys/content_archive.h" #include "core/file_sys/vfs_offset.h" #include "core/loader/loader.h" +#include "romfs.h" namespace FileSys { @@ -46,21 +47,9 @@ struct PFS0Superblock { }; static_assert(sizeof(PFS0Superblock) == 0x200, "PFS0Superblock has incorrect size."); -struct IVFCLevel { - u64_le offset; - u64_le size; - u32_le block_size; - u32_le reserved; -}; -static_assert(sizeof(IVFCLevel) == 0x18, "IVFCLevel has incorrect size."); - struct RomFSSuperblock { NCASectionHeaderBlock header_block; - u32_le magic; - u32_le magic_number; - INSERT_PADDING_BYTES(8); - std::array levels; - INSERT_PADDING_BYTES(64); + IVFCHeader ivfc; }; static_assert(sizeof(RomFSSuperblock) == 0xE8, "RomFSSuperblock has incorrect size."); @@ -92,8 +81,8 @@ NCA::NCA(VirtualFile file_) : file(std::move(file_)) { const size_t romfs_offset = header.section_tables[i].media_offset * MEDIA_OFFSET_MULTIPLIER + - sb.levels[IVFC_MAX_LEVEL - 1].offset; - const size_t romfs_size = sb.levels[IVFC_MAX_LEVEL - 1].size; + sb.ivfc.levels[IVFC_MAX_LEVEL - 1].offset; + const size_t romfs_size = sb.ivfc.levels[IVFC_MAX_LEVEL - 1].size; files.emplace_back(std::make_shared(file, romfs_size, romfs_offset)); romfs = files.back(); } else if (block.filesystem_type == NCASectionFilesystemType::PFS0) { diff --git a/src/core/file_sys/romfs.cpp b/src/core/file_sys/romfs.cpp new file mode 100644 index 000000000..ff3ddb29c --- /dev/null +++ b/src/core/file_sys/romfs.cpp @@ -0,0 +1,124 @@ +// Copyright 2018 yuzu emulator team +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#include "common/common_types.h" +#include "common/swap.h" +#include "core/file_sys/romfs.h" +#include "core/file_sys/vfs.h" +#include "core/file_sys/vfs_offset.h" +#include "core/file_sys/vfs_vector.h" + +namespace FileSys { + +constexpr u32 ROMFS_ENTRY_EMPTY = 0xFFFFFFFF; + +struct TableLocation { + u64_le offset; + u64_le size; +}; +static_assert(sizeof(TableLocation) == 0x10, "TableLocation has incorrect size."); + +struct RomFSHeader { + u64_le header_size; + TableLocation directory_hash; + TableLocation directory_meta; + TableLocation file_hash; + TableLocation file_meta; + u64_le data_offset; +}; +static_assert(sizeof(RomFSHeader) == 0x50, "RomFSHeader has incorrect size."); + +struct DirectoryEntry { + u32_le sibling; + u32_le child_dir; + u32_le child_file; + u32_le hash; + u32_le name_length; +}; +static_assert(sizeof(DirectoryEntry) == 0x14, "DirectoryEntry has incorrect size."); + +struct FileEntry { + u32_le parent; + u32_le sibling; + u64_le offset; + u64_le size; + u32_le hash; + u32_le name_length; +}; +static_assert(sizeof(FileEntry) == 0x20, "FileEntry has incorrect size."); + +template +static std::pair GetEntry(const VirtualFile& file, size_t offset) { + Entry entry{}; + if (file->ReadObject(&entry, offset) != sizeof(Entry)) + return {}; + std::string string(entry.name_length, '\0'); + if (file->ReadArray(&string[0], string.size(), offset + sizeof(Entry)) != string.size()) + return {}; + return {entry, string}; +} + +void ProcessFile(VirtualFile file, size_t file_offset, size_t data_offset, u32 this_file_offset, + std::shared_ptr parent) { + while (true) { + auto entry = GetEntry(file, file_offset + this_file_offset); + + parent->AddFile(std::make_shared( + file, entry.first.size, entry.first.offset + data_offset, entry.second, parent)); + + if (entry.first.sibling == ROMFS_ENTRY_EMPTY) + break; + + this_file_offset = entry.first.sibling; + } +} + +void ProcessDirectory(VirtualFile file, size_t dir_offset, size_t file_offset, size_t data_offset, + u32 this_dir_offset, std::shared_ptr parent) { + while (true) { + auto entry = GetEntry(file, dir_offset + this_dir_offset); + auto current = std::make_shared( + std::vector{}, std::vector{}, parent, entry.second); + + if (entry.first.child_file != ROMFS_ENTRY_EMPTY) { + ProcessFile(file, file_offset, data_offset, entry.first.child_file, current); + } + + if (entry.first.child_dir != ROMFS_ENTRY_EMPTY) { + ProcessDirectory(file, dir_offset, file_offset, data_offset, entry.first.child_dir, + current); + } + + parent->AddDirectory(current); + if (entry.first.sibling == ROMFS_ENTRY_EMPTY) + break; + this_dir_offset = entry.first.sibling; + } +} + +VirtualDir ExtractRomFS(VirtualFile file) { + RomFSHeader header{}; + if (file->ReadObject(&header) != sizeof(RomFSHeader)) + return nullptr; + + if (header.header_size != sizeof(RomFSHeader)) + return nullptr; + + const u64 file_offset = header.file_meta.offset; + const u64 dir_offset = header.directory_meta.offset + 4; + + const auto root = + std::make_shared(std::vector{}, std::vector{}, + file->GetContainingDirectory(), file->GetName()); + + ProcessDirectory(file, dir_offset, file_offset, header.data_offset, 0, root); + + VirtualDir out = std::move(root); + + while (out->GetSubdirectory("") != nullptr) + out = out->GetSubdirectory(""); + + return out; +} +} // namespace FileSys diff --git a/src/core/file_sys/romfs.h b/src/core/file_sys/romfs.h new file mode 100644 index 000000000..03a876d22 --- /dev/null +++ b/src/core/file_sys/romfs.h @@ -0,0 +1,35 @@ +// Copyright 2018 yuzu emulator team +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#pragma once + +#include +#include "common/common_funcs.h" +#include "common/swap.h" +#include "core/file_sys/vfs.h" + +namespace FileSys { + +struct IVFCLevel { + u64_le offset; + u64_le size; + u32_le block_size; + u32_le reserved; +}; +static_assert(sizeof(IVFCLevel) == 0x18, "IVFCLevel has incorrect size."); + +struct IVFCHeader { + u32_le magic; + u32_le magic_number; + INSERT_PADDING_BYTES(8); + std::array levels; + INSERT_PADDING_BYTES(64); +}; +static_assert(sizeof(IVFCHeader) == 0xE0, "IVFCHeader has incorrect size."); + +// Converts a RomFS binary blob to VFS Filesystem +// Returns nullptr on failure +VirtualDir ExtractRomFS(VirtualFile file); + +} // namespace FileSys diff --git a/src/core/file_sys/vfs.cpp b/src/core/file_sys/vfs.cpp index b99a4fd5b..84a6a7397 100644 --- a/src/core/file_sys/vfs.cpp +++ b/src/core/file_sys/vfs.cpp @@ -46,6 +46,13 @@ size_t VfsFile::WriteBytes(const std::vector& data, size_t offset) { return Write(data.data(), data.size(), offset); } +std::string VfsFile::GetFullPath() const { + if (GetContainingDirectory() == nullptr) + return "/" + GetName(); + + return GetContainingDirectory()->GetFullPath() + "/" + GetName(); +} + std::shared_ptr VfsDirectory::GetFileRelative(std::string_view path) const { auto vec = FileUtil::SplitPathComponents(path); vec.erase(std::remove_if(vec.begin(), vec.end(), [](const auto& str) { return str.empty(); }), @@ -243,6 +250,13 @@ bool VfsDirectory::Copy(std::string_view src, std::string_view dest) { return f2->WriteBytes(f1->ReadAllBytes()) == f1->GetSize(); } +std::string VfsDirectory::GetFullPath() const { + if (IsRoot()) + return GetName(); + + return GetParentDirectory()->GetFullPath() + "/" + GetName(); +} + bool ReadOnlyVfsDirectory::IsWritable() const { return false; } @@ -270,4 +284,13 @@ bool ReadOnlyVfsDirectory::DeleteFile(std::string_view name) { bool ReadOnlyVfsDirectory::Rename(std::string_view name) { return false; } + +bool VfsRawCopy(VirtualFile src, VirtualFile dest) { + if (src == nullptr || dest == nullptr) + return false; + if (!dest->Resize(src->GetSize())) + return false; + std::vector data = src->ReadAllBytes(); + return dest->WriteBytes(data, 0) == data.size(); +} } // namespace FileSys diff --git a/src/core/file_sys/vfs.h b/src/core/file_sys/vfs.h index 4a13b8378..cf871edd6 100644 --- a/src/core/file_sys/vfs.h +++ b/src/core/file_sys/vfs.h @@ -113,6 +113,9 @@ struct VfsFile : NonCopyable { // Renames the file to name. Returns whether or not the operation was successsful. virtual bool Rename(std::string_view name) = 0; + + // Returns the full path of this file as a string, recursively + virtual std::string GetFullPath() const; }; // A class representing a directory in an abstract filesystem. @@ -213,6 +216,17 @@ struct VfsDirectory : NonCopyable { return ReplaceFileWithSubdirectory(file_p, std::make_shared(file_p)); } + bool InterpretAsDirectory(const std::function& function, + const std::string& file) { + auto file_p = GetFile(file); + if (file_p == nullptr) + return false; + return ReplaceFileWithSubdirectory(file_p, function(file_p)); + } + + // Returns the full path of this directory as a string, recursively + virtual std::string GetFullPath() const; + protected: // Backend for InterpretAsDirectory. // Removes all references to file and adds a reference to dir in the directory's implementation. @@ -230,4 +244,10 @@ struct ReadOnlyVfsDirectory : public VfsDirectory { bool DeleteFile(std::string_view name) override; bool Rename(std::string_view name) override; }; + +// A method that copies the raw data between two different implementations of VirtualFile. If you +// are using the same implementation, it is probably better to use the Copy method in the parent +// directory of src/dest. +bool VfsRawCopy(VirtualFile src, VirtualFile dest); + } // namespace FileSys diff --git a/src/core/file_sys/vfs_offset.cpp b/src/core/file_sys/vfs_offset.cpp index a40331cef..847cde2f5 100644 --- a/src/core/file_sys/vfs_offset.cpp +++ b/src/core/file_sys/vfs_offset.cpp @@ -10,8 +10,9 @@ namespace FileSys { OffsetVfsFile::OffsetVfsFile(std::shared_ptr file_, size_t size_, size_t offset_, - std::string name_) - : file(std::move(file_)), offset(offset_), size(size_), name(std::move(name_)) {} + std::string name_, VirtualDir parent_) + : file(file_), offset(offset_), size(size_), name(std::move(name_)), + parent(parent_ == nullptr ? file->GetContainingDirectory() : std::move(parent_)) {} std::string OffsetVfsFile::GetName() const { return name.empty() ? file->GetName() : name; @@ -35,7 +36,7 @@ bool OffsetVfsFile::Resize(size_t new_size) { } std::shared_ptr OffsetVfsFile::GetContainingDirectory() const { - return file->GetContainingDirectory(); + return parent; } bool OffsetVfsFile::IsWritable() const { diff --git a/src/core/file_sys/vfs_offset.h b/src/core/file_sys/vfs_offset.h index 4f471e3ba..235970dc5 100644 --- a/src/core/file_sys/vfs_offset.h +++ b/src/core/file_sys/vfs_offset.h @@ -17,7 +17,7 @@ namespace FileSys { // the size of this wrapper. struct OffsetVfsFile : public VfsFile { OffsetVfsFile(std::shared_ptr file, size_t size, size_t offset = 0, - std::string new_name = ""); + std::string new_name = "", VirtualDir new_parent = nullptr); std::string GetName() const override; size_t GetSize() const override; @@ -44,6 +44,7 @@ private: size_t offset; size_t size; std::string name; + VirtualDir parent; }; } // namespace FileSys diff --git a/src/core/file_sys/vfs_real.cpp b/src/core/file_sys/vfs_real.cpp index 9ce2e1efa..82d54da4a 100644 --- a/src/core/file_sys/vfs_real.cpp +++ b/src/core/file_sys/vfs_real.cpp @@ -195,6 +195,12 @@ bool RealVfsDirectory::Rename(std::string_view name) { return FileUtil::Rename(path, new_name); } +std::string RealVfsDirectory::GetFullPath() const { + auto out = path; + std::replace(out.begin(), out.end(), '\\', '/'); + return out; +} + bool RealVfsDirectory::ReplaceFileWithSubdirectory(VirtualFile file, VirtualDir dir) { const auto iter = std::find(files.begin(), files.end(), file); if (iter == files.end()) diff --git a/src/core/file_sys/vfs_real.h b/src/core/file_sys/vfs_real.h index 2151211c9..243d58576 100644 --- a/src/core/file_sys/vfs_real.h +++ b/src/core/file_sys/vfs_real.h @@ -41,7 +41,7 @@ private: // An implementation of VfsDirectory that represents a directory on the user's computer. struct RealVfsDirectory : public VfsDirectory { - RealVfsDirectory(const std::string& path, Mode perms); + RealVfsDirectory(const std::string& path, Mode perms = Mode::Read); std::vector> GetFiles() const override; std::vector> GetSubdirectories() const override; @@ -54,6 +54,7 @@ struct RealVfsDirectory : public VfsDirectory { bool DeleteSubdirectory(std::string_view name) override; bool DeleteFile(std::string_view name) override; bool Rename(std::string_view name) override; + std::string GetFullPath() const override; protected: bool ReplaceFileWithSubdirectory(VirtualFile file, VirtualDir dir) override; diff --git a/src/core/file_sys/vfs_vector.cpp b/src/core/file_sys/vfs_vector.cpp new file mode 100644 index 000000000..4c6337e3a --- /dev/null +++ b/src/core/file_sys/vfs_vector.cpp @@ -0,0 +1,83 @@ +// Copyright 2018 yuzu emulator team +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#include +#include "core/file_sys/vfs_vector.h" + +namespace FileSys { +VectorVfsDirectory::VectorVfsDirectory(std::vector files_, + std::vector dirs_, VirtualDir parent_, + std::string name_) + : files(std::move(files_)), dirs(std::move(dirs_)), parent(std::move(parent_)), + name(std::move(name_)) {} + +std::vector> VectorVfsDirectory::GetFiles() const { + return files; +} + +std::vector> VectorVfsDirectory::GetSubdirectories() const { + return dirs; +} + +bool VectorVfsDirectory::IsWritable() const { + return false; +} + +bool VectorVfsDirectory::IsReadable() const { + return true; +} + +std::string VectorVfsDirectory::GetName() const { + return name; +} +std::shared_ptr VectorVfsDirectory::GetParentDirectory() const { + return parent; +} + +template +static bool FindAndRemoveVectorElement(std::vector& vec, std::string_view name) { + auto iter = std::find_if(vec.begin(), vec.end(), [name](T e) { return e->GetName() == name; }); + if (iter == vec.end()) + return false; + auto old_size = vec.size(); + vec.erase(iter); + return true; +} + +bool VectorVfsDirectory::DeleteSubdirectory(std::string_view name) { + return FindAndRemoveVectorElement(dirs, name); +} + +bool VectorVfsDirectory::DeleteFile(std::string_view name) { + return FindAndRemoveVectorElement(files, name); +} + +bool VectorVfsDirectory::Rename(std::string_view name_) { + name = name_; + return true; +} + +std::shared_ptr VectorVfsDirectory::CreateSubdirectory(std::string_view name) { + return nullptr; +} + +std::shared_ptr VectorVfsDirectory::CreateFile(std::string_view name) { + return nullptr; +} + +void VectorVfsDirectory::AddFile(VirtualFile file) { + files.push_back(std::move(file)); +} + +void VectorVfsDirectory::AddDirectory(VirtualDir dir) { + dirs.push_back(std::move(dir)); +} + +bool VectorVfsDirectory::ReplaceFileWithSubdirectory(VirtualFile file, VirtualDir dir) { + if (!DeleteFile(file->GetName())) + return false; + dirs.emplace_back(dir); + return true; +} +} // namespace FileSys diff --git a/src/core/file_sys/vfs_vector.h b/src/core/file_sys/vfs_vector.h new file mode 100644 index 000000000..ba469647b --- /dev/null +++ b/src/core/file_sys/vfs_vector.h @@ -0,0 +1,44 @@ +// Copyright 2018 yuzu emulator team +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#pragma once + +#include "core/file_sys/vfs.h" + +namespace FileSys { + +// An implementation of VfsDirectory that maintains two vectors for subdirectories and files. +// Vector data is supplied upon construction. +struct VectorVfsDirectory : public VfsDirectory { + explicit VectorVfsDirectory(std::vector files = {}, + std::vector dirs = {}, VirtualDir parent = nullptr, + std::string name = ""); + + std::vector> GetFiles() const override; + std::vector> GetSubdirectories() const override; + bool IsWritable() const override; + bool IsReadable() const override; + std::string GetName() const override; + std::shared_ptr GetParentDirectory() const override; + bool DeleteSubdirectory(std::string_view name) override; + bool DeleteFile(std::string_view name) override; + bool Rename(std::string_view name) override; + std::shared_ptr CreateSubdirectory(std::string_view name) override; + std::shared_ptr CreateFile(std::string_view name) override; + + virtual void AddFile(VirtualFile file); + virtual void AddDirectory(VirtualDir dir); + +protected: + bool ReplaceFileWithSubdirectory(VirtualFile file, VirtualDir dir) override; + +private: + std::vector files; + std::vector dirs; + + VirtualDir parent; + std::string name; +}; + +} // namespace FileSys