Mirrors/yuzu
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

file_sys: Add RegisteredCache

Browse source 
Manages NAND NCA get and install.
This commit is contained in:
Zach Hilman 2018-08-09 20:52:27 -04:00
parent 9aab787122
commit a91983b11c
2 changed files with 543 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

435
src/core/file_sys/registered_cache.cpp Normal file
View file

@ -0,0 +1,435 @@
// Copyright 2018 yuzu emulator team
// Licensed under GPLv2 or any later version
// Refer to the license.txt file included.
#include <regex>
#include <mbedtls/sha256.h>
#include "common/assert.h"
#include "common/hex_util.h"
#include "common/logging/log.h"
#include "core/crypto/encryption_layer.h"
#include "core/file_sys/card_image.h"
#include "core/file_sys/nca_metadata.h"
#include "core/file_sys/registered_cache.h"
#include "core/file_sys/vfs_concat.h"
namespace FileSys {
std::string RegisteredCacheEntry::DebugInfo() const {
return fmt::format("title_id={:016X}, content_type={:02X}", title_id, static_cast<u8>(type));
}
bool operator<(const RegisteredCacheEntry& lhs, const RegisteredCacheEntry& rhs) {
return (lhs.title_id < rhs.title_id) || (lhs.title_id == rhs.title_id && lhs.type < rhs.type);
}
static bool FollowsTwoDigitDirFormat(std::string_view name) {
const static std::regex two_digit_regex(
"000000[0123456789abcdefABCDEF][0123456789abcdefABCDEF]");
return std::regex_match(name.begin(), name.end(), two_digit_regex);
}
static bool FollowsNcaIdFormat(std::string_view name) {
const static std::regex nca_id_regex("[0123456789abcdefABCDEF]+.nca");
return name.size() == 36 && std::regex_match(name.begin(), name.end(), nca_id_regex);
}
static std::string GetRelativePathFromNcaID(const std::array<u8, 16>& nca_id, bool second_hex_upper,
bool within_two_digit) {
if (!within_two_digit)
return fmt::format("/{}.nca", HexArrayToString(nca_id, second_hex_upper));
Core::Crypto::SHA256Hash hash{};
mbedtls_sha256(nca_id.data(), nca_id.size(), hash.data(), 0);
return fmt::format("/000000{:02X}/{}.nca", hash[0], HexArrayToString(nca_id, second_hex_upper));
}
static std::string GetCNMTName(TitleType type, u64 title_id) {
constexpr std::array<const char*, 9> TITLE_TYPE_NAMES{
"SystemProgram",
"SystemData",
"SystemUpdate",
"BootImagePackage",
"BootImagePackageSafe",
"Application",
"Patch",
"AddOnContent",
"" ///< Currently unknown 'DeltaTitle'
};
size_t index = static_cast<size_t>(type);
if (index >= 0x80)
index -= 0x80;
return fmt::format("{}_{:016x}.cnmt", TITLE_TYPE_NAMES[index], title_id);
}
static ContentRecordType GetCRTypeFromNCAType(NCAContentType type) {
switch (type) {
case NCAContentType::Program:
// TODO(DarkLordZach): Differentiate between Program and Patch
return ContentRecordType::Program;
case NCAContentType::Meta:
return ContentRecordType::Meta;
case NCAContentType::Control:
return ContentRecordType::Control;
case NCAContentType::Data:
return ContentRecordType::Data;
case NCAContentType::Manual:
// TODO(DarkLordZach): Peek at NCA contents to differentiate Manual and Legal.
return ContentRecordType::Manual;
default:
UNREACHABLE();
}
}
VirtualFile RegisteredCache::OpenFileOrDirectoryConcat(const VirtualDir& dir,
std::string_view path) const {
if (dir->GetFileRelative(path) != nullptr)
return dir->GetFileRelative(path);
if (dir->GetDirectoryRelative(path) != nullptr) {
const auto nca_dir = dir->GetDirectoryRelative(path);
VirtualFile file = nullptr;
const auto files = nca_dir->GetFiles();
if (files.size() == 1 && files[0]->GetName() == "00")
file = files[0];
else {
std::vector<VirtualFile> concat;
for (u8 i = 0; i < 0x10; ++i) {
auto next = nca_dir->GetFile(fmt::format("{:02X}", i));
if (next != nullptr)
concat.push_back(std::move(next));
else {
next = nca_dir->GetFile(fmt::format("{:02x}", i));
if (next != nullptr)
concat.push_back(std::move(next));
else
break;
}
}
if (concat.empty())
return nullptr;
file = FileSys::ConcatenateFiles(concat);
}
return file;
}
return nullptr;
}
VirtualFile RegisteredCache::GetFileAtID(NcaID id) const {
VirtualFile file;
for (u8 i = 0; i < 4; ++i) {
file = OpenFileOrDirectoryConcat(
dir, GetRelativePathFromNcaID(id, (i & 0b10) == 0, (i & 0b01) == 0));
if (file != nullptr)
return file;
}
return file;
}
boost::optional<NcaID> RegisteredCache::GetNcaIDFromMetadata(u64 title_id,
ContentRecordType type) const {
if (type == ContentRecordType::Meta && meta_id.find(title_id) != meta_id.end())
return meta_id.at(title_id);
if (meta.find(title_id) == meta.end())
return boost::none;
const auto& cnmt = meta.at(title_id);
const auto iter = std::find_if(cnmt.GetContentRecords().begin(), cnmt.GetContentRecords().end(),
[type](const ContentRecord& rec) { return rec.type == type; });
if (iter == cnmt.GetContentRecords().end())
return boost::none;
return boost::make_optional(iter->nca_id);
}
void RegisteredCache::AccumulateFiles(std::vector<NcaID>& ids) const {
for (const auto& d2_dir : dir->GetSubdirectories()) {
if (FollowsNcaIdFormat(d2_dir->GetName())) {
ids.push_back(HexStringToArray<0x10, true>(d2_dir->GetName().substr(0, 0x20)));
continue;
}
if (!FollowsTwoDigitDirFormat(d2_dir->GetName()))
continue;
for (const auto& nca_dir : d2_dir->GetSubdirectories()) {
if (!FollowsNcaIdFormat(nca_dir->GetName()))
continue;
ids.push_back(HexStringToArray<0x10, true>(nca_dir->GetName().substr(0, 0x20)));
}
for (const auto& nca_file : d2_dir->GetFiles()) {
if (!FollowsNcaIdFormat(nca_file->GetName()))
continue;
ids.push_back(HexStringToArray<0x10, true>(nca_file->GetName().substr(0, 0x20)));
}
}
for (const auto& d2_file : dir->GetFiles()) {
if (FollowsNcaIdFormat(d2_file->GetName()))
ids.push_back(HexStringToArray<0x10, true>(d2_file->GetName().substr(0, 0x20)));
}
}
void RegisteredCache::ProcessFiles(const std::vector<NcaID>& ids) {
for (const auto& id : ids) {
const auto file = GetFileAtID(id);
if (file == nullptr)
continue;
const auto nca = std::make_shared<NCA>(parser(file, id));
if (nca->GetStatus() != Loader::ResultStatus::Success ||
nca->GetType() != NCAContentType::Meta)
continue;
const auto section0 = nca->GetSubdirectories()[0];
for (const auto& file : section0->GetFiles()) {
if (file->GetExtension() != "cnmt")
continue;
meta.insert_or_assign(nca->GetTitleId(), CNMT(file));
meta_id.insert_or_assign(nca->GetTitleId(), id);
break;
}
}
}
void RegisteredCache::AccumulateYuzuMeta() {
const auto dir = this->dir->GetSubdirectory("yuzu_meta");
if (dir == nullptr)
return;
for (const auto& file : dir->GetFiles()) {
if (file->GetExtension() != "cnmt")
continue;
CNMT cnmt(file);
yuzu_meta.insert_or_assign(cnmt.GetTitleID(), std::move(cnmt));
}
}
void RegisteredCache::Refresh() {
if (dir == nullptr)
return;
std::vector<NcaID> ids;
AccumulateFiles(ids);
ProcessFiles(ids);
AccumulateYuzuMeta();
}
RegisteredCache::RegisteredCache(VirtualDir dir_, RegisteredCacheParsingFunction parsing_function)
: dir(std::move(dir_)), parser(std::move(parsing_function)) {
Refresh();
}
bool RegisteredCache::HasEntry(u64 title_id, ContentRecordType type) const {
return GetEntryRaw(title_id, type) != nullptr;
}
bool RegisteredCache::HasEntry(RegisteredCacheEntry entry) const {
return GetEntryRaw(entry) != nullptr;
}
VirtualFile RegisteredCache::GetEntryRaw(u64 title_id, ContentRecordType type) const {
const auto id = GetNcaIDFromMetadata(title_id, type);
if (id == boost::none)
return nullptr;
return parser(GetFileAtID(id.get()), id.get());
}
VirtualFile RegisteredCache::GetEntryRaw(RegisteredCacheEntry entry) const {
return GetEntryRaw(entry.title_id, entry.type);
}
std::shared_ptr<NCA> RegisteredCache::GetEntry(u64 title_id, ContentRecordType type) const {
const auto raw = GetEntryRaw(title_id, type);
if (raw == nullptr)
return nullptr;
return std::make_shared<NCA>(raw);
}
std::shared_ptr<NCA> RegisteredCache::GetEntry(RegisteredCacheEntry entry) const {
return GetEntry(entry.title_id, entry.type);
}
template <typename T>
void RegisteredCache::IterateAllMetadata(
std::vector<T>& out, std::function<T(const CNMT&, const ContentRecord&)> proc,
std::function<bool(const CNMT&, const ContentRecord&)> filter) const {
for (const auto& kv : meta) {
const auto& cnmt = kv.second;
if (filter(cnmt, EMPTY_META_CONTENT_RECORD))
out.push_back(proc(cnmt, EMPTY_META_CONTENT_RECORD));
for (const auto& rec : cnmt.GetContentRecords()) {
if (GetFileAtID(rec.nca_id) != nullptr && filter(cnmt, rec)) {
out.push_back(proc(cnmt, rec));
}
}
}
for (const auto& kv : yuzu_meta) {
const auto& cnmt = kv.second;
for (const auto& rec : cnmt.GetContentRecords()) {
if (GetFileAtID(rec.nca_id) != nullptr && filter(cnmt, rec)) {
out.push_back(proc(cnmt, rec));
}
}
}
}
std::vector<RegisteredCacheEntry> RegisteredCache::ListEntries() const {
std::vector<RegisteredCacheEntry> out;
IterateAllMetadata<RegisteredCacheEntry>(
out,
[](const CNMT& c, const ContentRecord& r) {
return RegisteredCacheEntry{c.GetTitleID(), r.type};
},
[](const CNMT& c, const ContentRecord& r) { return true; });
return out;
}
std::vector<RegisteredCacheEntry> RegisteredCache::ListEntriesFilter(
boost::optional<TitleType> title_type, boost::optional<ContentRecordType> record_type,
boost::optional<u64> title_id) const {
std::vector<RegisteredCacheEntry> out;
IterateAllMetadata<RegisteredCacheEntry>(
out,
[](const CNMT& c, const ContentRecord& r) {
return RegisteredCacheEntry{c.GetTitleID(), r.type};
},
[&title_type, &record_type, &title_id](const CNMT& c, const ContentRecord& r) {
if (title_type != boost::none && title_type.get() != c.GetType())
return false;
if (record_type != boost::none && record_type.get() != r.type)
return false;
if (title_id != boost::none && title_id.get() != c.GetTitleID())
return false;
return true;
});
return out;
}
static std::shared_ptr<NCA> GetNCAFromXCIForID(std::shared_ptr<XCI> xci, const NcaID& id) {
const auto filename = fmt::format("{}.nca", HexArrayToString(id, false));
const auto iter =
std::find_if(xci->GetNCAs().begin(), xci->GetNCAs().end(),
[&filename](std::shared_ptr<NCA> nca) { return nca->GetName() == filename; });
return iter == xci->GetNCAs().end() ? nullptr : *iter;
}
bool RegisteredCache::InstallEntry(std::shared_ptr<XCI> xci) {
const auto& ncas = xci->GetNCAs();
const auto& meta_iter = std::find_if(ncas.begin(), ncas.end(), [](std::shared_ptr<NCA> nca) {
return nca->GetType() == NCAContentType::Meta;
});
if (meta_iter == ncas.end()) {
LOG_ERROR(Loader, "The XCI you are attempting to install does not have a metadata NCA and "
"is therefore malformed. Double check your encryption keys.");
return false;
}
// Install Metadata File
const auto meta_id_raw = (*meta_iter)->GetName().substr(0, 32);
const auto meta_id = HexStringToArray<16>(meta_id_raw);
if (!RawInstallNCA(*meta_iter, meta_id))
return false;
// Install all the other NCAs
const auto section0 = (*meta_iter)->GetSubdirectories()[0];
const auto cnmt_file = section0->GetFiles()[0];
const CNMT cnmt(cnmt_file);
for (const auto& record : cnmt.GetContentRecords()) {
const auto nca = GetNCAFromXCIForID(xci, record.nca_id);
if (nca == nullptr || !RawInstallNCA(nca, record.nca_id))
return false;
}
Refresh();
return true;
}
bool RegisteredCache::InstallEntry(std::shared_ptr<NCA> nca, TitleType type) {
CNMTHeader header{
nca->GetTitleId(), ///< Title ID
0, ///< Ignore/Default title version
type, ///< Type
{}, ///< Padding
0x10, ///< Default table offset
1, ///< 1 Content Entry
0, ///< No Meta Entries
{}, ///< Padding
};
OptionalHeader opt_header{0, 0};
ContentRecord c_rec{{}, {}, {}, GetCRTypeFromNCAType(nca->GetType()), {}};
const auto& data = nca->GetBaseFile()->ReadBytes(0x100000);
mbedtls_sha256(data.data(), data.size(), c_rec.hash.data(), 0);
memcpy(&c_rec.nca_id, &c_rec.hash, 16);
const CNMT new_cnmt(header, opt_header, {c_rec}, {});
return RawInstallYuzuMeta(new_cnmt) && RawInstallNCA(nca, c_rec.nca_id);
}
bool RegisteredCache::RawInstallNCA(std::shared_ptr<NCA> nca, boost::optional<NcaID> override_id) {
const auto in = nca->GetBaseFile();
Core::Crypto::SHA256Hash hash{};
// Calculate NcaID
// NOTE: Because computing the SHA256 of an entire NCA is quite expensive (especially if the
// game is massive), we're going to cheat and only hash the first MB of the NCA.
// Also, for XCIs the NcaID matters, so if the override id isn't none, use that.
NcaID id{};
if (override_id == boost::none) {
const auto& data = in->ReadBytes(0x100000);
mbedtls_sha256(data.data(), data.size(), hash.data(), 0);
memcpy(id.data(), hash.data(), 16);
} else {
id = override_id.get();
}
std::string path = GetRelativePathFromNcaID(id, false, true);
if (GetFileAtID(id) != nullptr) {
LOG_WARNING(Loader, "OW Attempt");
return false;
}
auto out = dir->CreateFileRelative(path);
if (out == nullptr)
return false;
return VfsRawCopy(in, out);
}
bool RegisteredCache::RawInstallYuzuMeta(const CNMT& cnmt) {
const auto dir = this->dir->CreateDirectoryRelative("yuzu_meta");
const auto filename = GetCNMTName(cnmt.GetType(), cnmt.GetTitleID());
if (dir->GetFile(filename) == nullptr) {
auto out = dir->CreateFile(filename);
const auto buffer = cnmt.Serialize();
out->Resize(buffer.size());
out->WriteBytes(buffer);
} else {
auto out = dir->GetFile(filename);
CNMT old_cnmt(out);
// Returns true on change
if (old_cnmt.UnionRecords(cnmt)) {
out->Resize(0);
const auto buffer = old_cnmt.Serialize();
out->Resize(buffer.size());
out->WriteBytes(buffer);
}
}
Refresh();
return std::find_if(yuzu_meta.begin(), yuzu_meta.end(),
[&cnmt](const std::pair<const u64, CNMT>& kv) {
return kv.second.GetType() == cnmt.GetType() &&
kv.second.GetTitleID() == cnmt.GetTitleID();
}) != yuzu_meta.end();
}
} // namespace FileSys

108
src/core/file_sys/registered_cache.h Normal file
View file

@ -0,0 +1,108 @@
// Copyright 2018 yuzu emulator team
// Licensed under GPLv2 or any later version
// Refer to the license.txt file included.
#pragma once
#include <array>
#include <map>
#include <memory>
#include <string>
#include <boost/container/flat_map.hpp>
#include "common/common_funcs.h"
#include "content_archive.h"
#include "core/file_sys/vfs.h"
#include "nca_metadata.h"
namespace FileSys {
class XCI;
class CNMT;
using NcaID = std::array<u8, 0x10>;
using RegisteredCacheParsingFunction = std::function<VirtualFile(const VirtualFile&, const NcaID&)>;
struct RegisteredCacheEntry {
u64 title_id;
ContentRecordType type;
std::string DebugInfo() const;
};
// boost flat_map requires operator< for O(log(n)) lookups.
bool operator<(const RegisteredCacheEntry& lhs, const RegisteredCacheEntry& rhs);
/*
* A class that catalogues NCAs in the registered directory structure.
* Nintendo's registered format follows this structure:
*
* Root
* | 000000XX <- XX is the ____ two digits of the NcaID
* | <hash>.nca <- hash is the NcaID (first half of SHA256 over entire file) (folder)
* | 00
* | 01 <- Actual content split along 4GB boundaries. (optional)
*
* (This impl also supports substituting the nca dir for an nca file, as that's more convenient when
* 4GB splitting can be ignored.)
*/
class RegisteredCache {
public:
// Parsing function defines the conversion from raw file to NCA. If there are other steps
// besides creating the NCA from the file (e.g. NAX0 on SD Card), that should go in a custom
// parsing function.
RegisteredCache(VirtualDir dir,
RegisteredCacheParsingFunction parsing_function =
[](const VirtualFile& file, const NcaID& id) { return file; });
void Refresh();
bool HasEntry(u64 title_id, ContentRecordType type) const;
bool HasEntry(RegisteredCacheEntry entry) const;
VirtualFile GetEntryRaw(u64 title_id, ContentRecordType type) const;
VirtualFile GetEntryRaw(RegisteredCacheEntry entry) const;
std::shared_ptr<NCA> GetEntry(u64 title_id, ContentRecordType type) const;
std::shared_ptr<NCA> GetEntry(RegisteredCacheEntry entry) const;
std::vector<RegisteredCacheEntry> ListEntries() const;
// If a parameter is not boost::none, it will be filtered for from all entries.
std::vector<RegisteredCacheEntry> ListEntriesFilter(
boost::optional<TitleType> title_type = boost::none,
boost::optional<ContentRecordType> record_type = boost::none,
boost::optional<u64> title_id = boost::none) const;
// Raw copies all the ncas from the xci to the csache. Does some quick checks to make sure there
// is a meta NCA and all of them are accessible.
bool InstallEntry(std::shared_ptr<XCI> xci);
// Due to the fact that we must use Meta-type NCAs to determine the existance of files, this
// poses quite a challenge. Instead of creating a new meta NCA for this file, yuzu will create a
// dir inside the NAND called 'yuzu_meta' and store the raw CNMT there.
// TODO(DarkLordZach): Author real meta-type NCAs and install those.
bool InstallEntry(std::shared_ptr<NCA> nca, TitleType type);
private:
template <typename T>
void IterateAllMetadata(std::vector<T>& out,
std::function<T(const CNMT&, const ContentRecord&)> proc,
std::function<bool(const CNMT&, const ContentRecord&)> filter) const;
void AccumulateFiles(std::vector<NcaID>& ids) const;
void ProcessFiles(const std::vector<NcaID>& ids);
void AccumulateYuzuMeta();
boost::optional<NcaID> GetNcaIDFromMetadata(u64 title_id, ContentRecordType type) const;
VirtualFile GetFileAtID(NcaID id) const;
VirtualFile OpenFileOrDirectoryConcat(const VirtualDir& dir, std::string_view path) const;
bool RawInstallNCA(std::shared_ptr<NCA> nca, boost::optional<NcaID> override_id = boost::none);
bool RawInstallYuzuMeta(const CNMT& cnmt);
VirtualDir dir;
RegisteredCacheParsingFunction parser;
// maps tid -> NcaID of meta
boost::container::flat_map<u64, NcaID> meta_id;
// maps tid -> meta
boost::container::flat_map<u64, CNMT> meta;
// maps tid -> meta for CNMT in yuzu_meta
boost::container::flat_map<u64, CNMT> yuzu_meta;
};
} // namespace FileSys